From ff10e07c098ee5003a81f8dc0de23f8ba8158e78 Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Mon, 22 Jun 2026 12:17:41 +0300 Subject: [PATCH] vmcluster: support storage node pools --- api/operator/v1/vlagent_types.go | 4 +- api/operator/v1/vlcluster_types.go | 16 +- api/operator/v1/vlsingle_types.go | 10 +- api/operator/v1/vmanomaly_types.go | 4 +- api/operator/v1/vtcluster_types.go | 16 +- api/operator/v1/vtsingle_types.go | 4 +- api/operator/v1beta1/vlogs_types.go | 4 +- api/operator/v1beta1/vmagent_types.go | 4 +- api/operator/v1beta1/vmalert_types.go | 4 +- api/operator/v1beta1/vmalertmanager_types.go | 4 +- api/operator/v1beta1/vmcluster_types.go | 350 ++++++++++--- api/operator/v1beta1/vmcluster_types_test.go | 87 ++++ api/operator/v1beta1/vmextra_types.go | 56 ++ api/operator/v1beta1/vmextra_types_test.go | 140 +++++ api/operator/v1beta1/vmsingle_types.go | 10 +- api/operator/v1beta1/vmuser_types.go | 6 +- api/operator/v1beta1/zz_generated.deepcopy.go | 32 ++ config/crd/overlay/crd.descriptionless.yaml | 49 ++ config/crd/overlay/crd.yaml | 120 +++++ docs/CHANGELOG.md | 3 +- docs/api.md | 21 +- docs/resources/vmcluster.md | 116 +++++ .../operator/factory/build/build.go | 66 --- .../operator/factory/build/build_test.go | 147 ------ .../operator/factory/build/cluster.go | 26 +- .../operator/factory/vmauth/vmusers_config.go | 67 ++- .../factory/vmauth/vmusers_config_test.go | 88 +++- .../operator/factory/vmcluster/vmcluster.go | 488 +++++++++++------- .../factory/vmcluster/vmcluster_pools.go | 205 ++++++++ .../vmcluster/vmcluster_reconcile_test.go | 43 +- .../factory/vmcluster/vmcluster_test.go | 288 +++++++++++ internal/converter/converter.go | 3 +- internal/podutil/util.go | 4 +- 33 files changed, 1897 insertions(+), 588 deletions(-) create mode 100644 internal/controller/operator/factory/vmcluster/vmcluster_pools.go diff --git a/api/operator/v1/vlagent_types.go b/api/operator/v1/vlagent_types.go index fa439e990a..7c6524f6d7 100644 --- a/api/operator/v1/vlagent_types.go +++ b/api/operator/v1/vlagent_types.go @@ -484,13 +484,13 @@ func (cr *VLAgent) IsOwnsServiceAccount() bool { } // AsURL - returns url for http access -func (cr *VLAgent) AsURL(isExtra bool) string { +func (cr *VLAgent) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "9429" } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // ProbePath implements build.probeCRD interface diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go index 04c97daa10..875457892b 100644 --- a/api/operator/v1/vlcluster_types.go +++ b/api/operator/v1/vlcluster_types.go @@ -866,14 +866,14 @@ func (cr *VLCluster) IsOwnsServiceAccount() bool { // AsURL implements stub for interface. // nolint:dupl,lll -func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string { +func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) (string, error) { var defaultPort string var svcSpec *vmv1beta1.AdditionalServiceSpec var extraArgs map[string]string switch kind { case vmv1beta1.ClusterComponentSelect: if cr.Spec.VLSelect == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlSelect configured", cr.Name) } defaultPort = "9471" if cr.Spec.VLSelect.Port != "" { @@ -883,7 +883,7 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.VLSelect.ExtraArgs case vmv1beta1.ClusterComponentInsert: if cr.Spec.VLInsert == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlInsert configured", cr.Name) } defaultPort = "9481" if cr.Spec.VLInsert.Port != "" { @@ -893,7 +893,7 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.VLInsert.ExtraArgs case vmv1beta1.ClusterComponentStorage: if cr.Spec.VLStorage == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlStorage configured", cr.Name) } defaultPort = "9491" if cr.Spec.VLStorage.Port != "" { @@ -905,12 +905,16 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string panic("BUG unsupported cluster kind=" + string(kind)) } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port), nil } // GetRemoteWriteURL returns the insert URL for VLCluster (used by VLDistributed) func (cr *VLCluster) GetRemoteWriteURL() string { - return cr.AsURL(vmv1beta1.ClusterComponentInsert, false) + "/insert/native" + url, err := cr.AsURL(vmv1beta1.ClusterComponentInsert, false) + if err != nil { + return "" + } + return url + "/insert/native" } // +kubebuilder:object:root=true diff --git a/api/operator/v1/vlsingle_types.go b/api/operator/v1/vlsingle_types.go index 4cd7349b89..3c2ad1e284 100644 --- a/api/operator/v1/vlsingle_types.go +++ b/api/operator/v1/vlsingle_types.go @@ -337,13 +337,13 @@ func (cr *VLSingle) IsOwnsServiceAccount() bool { return cr.Spec.ServiceAccountName == "" } -func (cr *VLSingle) AsURL(isExtra bool) string { +func (cr *VLSingle) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "9428" } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated @@ -364,5 +364,9 @@ func (cr *VLSingle) GetAdditionalService() *vmv1beta1.AdditionalServiceSpec { // GetRemoteWriteURL returns the native insert URL for VLSingle (used by VLDistributed) func (cr *VLSingle) GetRemoteWriteURL() string { - return cr.AsURL(false) + "/insert/native" + url, err := cr.AsURL(false) + if err != nil { + return "" + } + return url + "/insert/native" } diff --git a/api/operator/v1/vmanomaly_types.go b/api/operator/v1/vmanomaly_types.go index 0b0667a8f8..2e2ae828ae 100644 --- a/api/operator/v1/vmanomaly_types.go +++ b/api/operator/v1/vmanomaly_types.go @@ -483,9 +483,9 @@ func (*VMAnomaly) ProbeNeedLiveness() bool { // AsURL returns url for http access to the first replica. // Returns empty string if spec.server.port is not configured. -func (cr *VMAnomaly) AsURL(isExtra bool) string { +func (cr *VMAnomaly) AsURL(isExtra bool) (string, error) { svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(), cr.Port(), "http", nil, isExtra) - return fmt.Sprintf("http://%s.%s.svc:%s", svcName, cr.Namespace, port) + return fmt.Sprintf("http://%s.%s.svc:%s", svcName, cr.Namespace, port), nil } // Validate performs semantic validation for component diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go index e7277c0ff2..3e4cde19d1 100644 --- a/api/operator/v1/vtcluster_types.go +++ b/api/operator/v1/vtcluster_types.go @@ -655,7 +655,11 @@ func (cr *VTCluster) Validate() error { } storageNodes := sets.New[string]() if cr.Spec.Storage != nil { - storageNodes.Insert(cr.AsURL(vmv1beta1.ClusterComponentStorage, false)) + storageURL, err := cr.AsURL(vmv1beta1.ClusterComponentStorage, false) + if err != nil { + return err + } + storageNodes.Insert(storageURL) vts := cr.Spec.Storage name := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) if vts.ServiceSpec != nil && vts.ServiceSpec.Name == name { @@ -777,14 +781,14 @@ func (cr *VTCluster) IsOwnsServiceAccount() bool { // AsURL implements stub for interface. // nolint:dupl,lll -func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string { +func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) (string, error) { var defaultPort string var svcSpec *vmv1beta1.AdditionalServiceSpec var extraArgs map[string]string switch kind { case vmv1beta1.ClusterComponentSelect: if cr.Spec.Select == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.select configured", cr.Name) } defaultPort = "10471" if cr.Spec.Select.Port != "" { @@ -794,7 +798,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.Select.ExtraArgs case vmv1beta1.ClusterComponentInsert: if cr.Spec.Insert == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.insert configured", cr.Name) } defaultPort = "10481" if cr.Spec.Insert.Port != "" { @@ -804,7 +808,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.Insert.ExtraArgs case vmv1beta1.ClusterComponentStorage: if cr.Spec.Storage == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.storage configured", cr.Name) } defaultPort = "10491" if cr.Spec.Storage.Port != "" { @@ -816,7 +820,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string panic("BUG unsupported cluster kind=" + string(kind)) } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port), nil } // +kubebuilder:object:root=true diff --git a/api/operator/v1/vtsingle_types.go b/api/operator/v1/vtsingle_types.go index 5881601d32..54bf43b1df 100644 --- a/api/operator/v1/vtsingle_types.go +++ b/api/operator/v1/vtsingle_types.go @@ -334,13 +334,13 @@ func (cr *VTSingle) IsOwnsServiceAccount() bool { } // AsURL returns URL for components access -func (cr *VTSingle) AsURL(isExtra bool) string { +func (cr *VTSingle) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "10428" } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated diff --git a/api/operator/v1beta1/vlogs_types.go b/api/operator/v1beta1/vlogs_types.go index 38bd094d5d..855ea0b355 100644 --- a/api/operator/v1beta1/vlogs_types.go +++ b/api/operator/v1beta1/vlogs_types.go @@ -299,13 +299,13 @@ func (cr *VLogs) IsOwnsServiceAccount() bool { return cr.Spec.ServiceAccountName == "" } -func (cr *VLogs) AsURL(isExtra bool) string { +func (cr *VLogs) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "9428" } svcName, port := ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated diff --git a/api/operator/v1beta1/vmagent_types.go b/api/operator/v1beta1/vmagent_types.go index 83deaa8a26..76b3c68082 100644 --- a/api/operator/v1beta1/vmagent_types.go +++ b/api/operator/v1beta1/vmagent_types.go @@ -612,13 +612,13 @@ func (cr *VMAgent) GetRBACName() string { } // AsURL - returns url for http access -func (cr *VMAgent) AsURL(isExtra bool) string { +func (cr *VMAgent) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "8429" } svcName, port := ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } func (cr *VMAgent) ProbePath() string { diff --git a/api/operator/v1beta1/vmalert_types.go b/api/operator/v1beta1/vmalert_types.go index 5eb14ae55a..35407e77d1 100644 --- a/api/operator/v1beta1/vmalert_types.go +++ b/api/operator/v1beta1/vmalert_types.go @@ -516,13 +516,13 @@ func (cr *VMAlert) IsOwnsServiceAccount() bool { return cr.Spec.ServiceAccountName == "" } -func (cr *VMAlert) AsURL(isExtra bool) string { +func (cr *VMAlert) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "8080" } svcName, port := ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // IsUnmanaged checks if object should managed any config objects diff --git a/api/operator/v1beta1/vmalertmanager_types.go b/api/operator/v1beta1/vmalertmanager_types.go index 73e12f3c1f..ca2048616f 100644 --- a/api/operator/v1beta1/vmalertmanager_types.go +++ b/api/operator/v1beta1/vmalertmanager_types.go @@ -404,13 +404,13 @@ func (cr *VMAlertmanager) Port() string { // AsURL returns url for accessing alertmanager // via corresponding service -func (cr *VMAlertmanager) AsURL(isExtra bool) string { +func (cr *VMAlertmanager) AsURL(isExtra bool) (string, error) { portName := cr.Spec.PortName if portName == "" { portName = "web" } svcName, port := ResolveServiceURL(cr.PrefixedName(), cr.Port(), portName, cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", cr.accessScheme(), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", cr.accessScheme(), svcName, cr.Namespace, port), nil } // returns fqdn for direct pod access diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index bbe65d8da9..a2ee7f90b6 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" ) @@ -100,6 +101,18 @@ type VMClusterSpec struct { // See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery // +optional Discovery *VMClusterDiscovery `json:"discovery,omitempty"` + + // Pools defines named groups of vmstorage (and optionally vminsert) components. + // Each pool gets its own StatefulSet and headless Service named --. + // Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field. + // vmselect queries all pools using the pool name as a storage group name (-storageNode=/). + // When pools are defined the top-level vmstorage is not deployed; pools replace it entirely. + // The top-level vminsert is deployed as a shared insert group across all pools only when no pool + // defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. + // +optional + // +listType=map + // +listMapKey=name + Pools []VMClusterPool `json:"pools,omitempty"` } // VMClusterDiscovery configures automatic vmstorage node discovery for vminsert and vmselect. @@ -134,7 +147,16 @@ func (d *VMClusterDiscovery) enabled() bool { return d != nil && d.Enabled } -func (d *VMClusterDiscovery) validate() error { +func (d *VMClusterDiscovery) validate(license *License) error { + if !d.enabled() { + return nil + } + if !license.IsProvided() { + return fmt.Errorf("discovery requires a valid license key, see https://docs.victoriametrics.com/victoriametrics/enterprise/") + } + if err := license.validate(); err != nil { + return err + } if len(d.Filter) > 0 { if _, err := regexp.Compile(d.Filter); err != nil { return fmt.Errorf("discovery.filter is not a valid regexp: %w", err) @@ -520,6 +542,11 @@ type VMStorage struct { // it can be overwritten with component specific image.tag value. // +optional ComponentVersion string `json:"componentVersion,omitempty"` + // RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance. + // Useful when using Pools to implement multi-retention setups. + // +optional + // +kubebuilder:validation:Pattern:="^[0-9]+(h|d|w|y)?$" + RetentionPeriod string `json:"retentionPeriod,omitempty"` // PodMetadata configures Labels and Annotations which are propagated to the VMStorage pods. PodMetadata *EmbeddedObjectMetadata `json:"podMetadata,omitempty"` // LogFormat for VMStorage to be configured with. @@ -738,10 +765,61 @@ func (cr *VMCluster) GetRemoteWriteURL() string { if cr == nil || cr.Spec.VMInsert == nil { return "" } - insertURL := cr.AsURL(ClusterComponentInsert, false) + insertURL, err := cr.AsURL(ClusterComponentInsert, "", false) + if err != nil { + return "" + } return fmt.Sprintf("%s%s", insertURL, BuildPathWithPrefixFlag(cr.Spec.VMInsert.ExtraArgs, "/insert/multitenant/prometheus/api/v1/write")) } +func (vms *VMStorage) validate(license *License, clusterRetentionPeriod string) error { + if vms.VMBackup != nil { + if err := vms.VMBackup.validate(license); err != nil { + return err + } + } + retention := clusterRetentionPeriod + if vms.RetentionPeriod != "" { + retention = vms.RetentionPeriod + } + if err := vms.RetentionFilters.validate(license, retention); err != nil { + return err + } + if vms.HPA != nil { + if vms.HPA.Behaviour != nil && vms.HPA.Behaviour.ScaleDown != nil { + return fmt.Errorf("scaledown HPA behavior is not supported") + } + if err := vms.HPA.Validate(); err != nil { + return err + } + } + if vms.VPA != nil { + if err := vms.VPA.Validate(); err != nil { + return err + } + } + if vms.RollingUpdateStrategyBehavior != nil { + if err := vms.RollingUpdateStrategyBehavior.Validate(); err != nil { + return err + } + } + return vms.Validate() +} + +func (vmi *VMInsert) validate() error { + if vmi.HPA != nil { + if err := vmi.HPA.Validate(); err != nil { + return err + } + } + if vmi.VPA != nil { + if err := vmi.VPA.Validate(); err != nil { + return err + } + } + return vmi.Validate() +} + func (cr *VMCluster) Validate() error { if MustSkipCRValidation(cr) { return nil @@ -802,17 +880,7 @@ func (cr *VMCluster) Validate() error { if vmi.ServiceSpec != nil && vmi.ServiceSpec.Name == name { return fmt.Errorf(".serviceSpec.Name cannot be equal to prefixed name=%q", name) } - if vmi.HPA != nil { - if err := vmi.HPA.Validate(); err != nil { - return err - } - } - if vmi.VPA != nil { - if err := vmi.VPA.Validate(); err != nil { - return err - } - } - if err := vmi.Validate(); err != nil { + if err := vmi.validate(); err != nil { return fmt.Errorf("vminsert: %w", err) } } @@ -825,28 +893,7 @@ func (cr *VMCluster) Validate() error { if vms.ServiceSpec != nil && vms.ServiceSpec.Name == name { return fmt.Errorf(".serviceSpec.Name cannot be equal to prefixed name=%q", name) } - if cr.Spec.VMStorage.VMBackup != nil { - if err := cr.Spec.VMStorage.VMBackup.validate(cr.Spec.License); err != nil { - return err - } - } - if err := vms.RetentionFilters.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { - return err - } - if vms.RollingUpdateStrategyBehavior != nil { - if err := vms.RollingUpdateStrategyBehavior.Validate(); err != nil { - return fmt.Errorf("vmstorage: %w", err) - } - } - if vms.HPA != nil && vms.HPA.Behaviour != nil && vms.HPA.Behaviour.ScaleDown != nil { - return fmt.Errorf("vmstorage scaledown HPA behavior is not supported") - } - if vms.VPA != nil { - if err := vms.VPA.Validate(); err != nil { - return err - } - } - if err := vms.Validate(); err != nil { + if err := vms.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { return fmt.Errorf("vmstorage: %w", err) } } @@ -867,28 +914,69 @@ func (cr *VMCluster) Validate() error { if cr.Spec.VMSelect != nil { vmselectDiscovery = cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery) } - if vminsertDiscovery.enabled() || vmselectDiscovery.enabled() { - if !cr.Spec.License.IsProvided() { - return fmt.Errorf("discovery requires a valid license key, see https://docs.victoriametrics.com/victoriametrics/enterprise/") - } - if err := cr.Spec.License.validate(); err != nil { - return err - } - } if vminsertDiscovery.enabled() { if cr.Spec.VMStorage != nil && len(cr.Spec.VMStorage.MaintenanceInsertNodeIDs) > 0 { return fmt.Errorf("maintenanceInsertNodeIDs cannot be used when vminsert discovery is enabled") } - if err := vminsertDiscovery.validate(); err != nil { - return fmt.Errorf("vminsert: %w", err) - } + } + if err := vminsertDiscovery.validate(cr.Spec.License); err != nil { + return fmt.Errorf("vminsert: %w", err) } if vmselectDiscovery.enabled() { if cr.Spec.VMStorage != nil && len(cr.Spec.VMStorage.MaintenanceSelectNodeIDs) > 0 { return fmt.Errorf("maintenanceSelectNodeIDs cannot be used when vmselect discovery is enabled") } - if err := vmselectDiscovery.validate(); err != nil { - return fmt.Errorf("vmselect: %w", err) + } + if err := vmselectDiscovery.validate(cr.Spec.License); err != nil { + return fmt.Errorf("vmselect: %w", err) + } + + poolNames := make(map[string]struct{}, len(cr.Spec.Pools)) + var hasPoolInsert bool + for i, pool := range cr.Spec.Pools { + if errs := validation.IsDNS1123Subdomain(pool.Name); len(errs) > 0 { + return fmt.Errorf("pools[%d].name %q is invalid: %s", i, pool.Name, strings.Join(errs, "; ")) + } + if len(pool.Name) > maxVMClusterPoolNameLength { + return fmt.Errorf("pools[%d].name %q is too long: max %d characters", i, pool.Name, maxVMClusterPoolNameLength) + } + if _, dup := poolNames[pool.Name]; dup { + return fmt.Errorf("pools[%d].name %q is duplicated", i, pool.Name) + } + poolNames[pool.Name] = struct{}{} + if pool.VMStorage != nil { + vms := pool.VMStorage.DeepCopy() + if cr.Spec.VMStorage != nil { + if err := MergeDeep(vms, cr.Spec.VMStorage, true); err != nil { + return fmt.Errorf("pools[%d] vmstorage merge: %w", i, err) + } + } + if err := vms.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { + return fmt.Errorf("pools[%d] vmstorage: %w", i, err) + } + } + if pool.VMInsert != nil { + hasPoolInsert = true + vmi := pool.VMInsert.DeepCopy() + if cr.Spec.VMInsert != nil { + if err := MergeDeep(vmi, cr.Spec.VMInsert, true); err != nil { + return fmt.Errorf("pools[%d] vminsert merge: %w", i, err) + } + } + if err := vmi.validate(); err != nil { + return fmt.Errorf("pools[%d] vminsert: %w", i, err) + } + poolDiscovery := vmi.Discovery.OrDefault(cr.Spec.Discovery) + if err := poolDiscovery.validate(cr.Spec.License); err != nil { + return fmt.Errorf("pools[%d] vminsert: %w", i, err) + } + } + } + if hasPoolInsert { + for i, pool := range cr.Spec.Pools { + if pool.VMInsert == nil { + return fmt.Errorf("pools[%d] %q: vminsert must be defined once any pool defines its own vminsert, otherwise this pool has no ingestion path", i, pool.Name) + } } } @@ -1051,15 +1139,73 @@ func (cr *VMCluster) IsOwnsServiceAccount() bool { return cr.Spec.ServiceAccountName == "" } -// AsURL implements stub for interface. -func (cr *VMCluster) AsURL(kind ClusterComponent, isExtra bool) string { +// findPool returns the pool with the given name, if any. +func (cr *VMCluster) findPool(name string) (*VMClusterPool, bool) { + for i := range cr.Spec.Pools { + if cr.Spec.Pools[i].Name == name { + return &cr.Spec.Pools[i], true + } + } + return nil, false +} + +// hasAnyPoolInsert reports whether any pool defines its own dedicated vminsert, which is +// exactly the condition under which the operator skips creating the shared top-level vminsert. +func hasAnyPoolInsert(pools []VMClusterPool) bool { + for _, p := range pools { + if p.VMInsert != nil { + return true + } + } + return false +} + +// mergedPoolInsert merges pool.VMInsert over the top-level base, mirroring how the operator +// resolves a pool's own vminsert when reconciling it. +func (cr *VMCluster) mergedPoolInsert(pool *VMClusterPool) (*VMInsert, error) { + merged := pool.VMInsert.DeepCopy() + if cr.Spec.VMInsert != nil { + if err := MergeDeep(merged, cr.Spec.VMInsert, true); err != nil { + return nil, fmt.Errorf("pool %q vminsert merge: %w", pool.Name, err) + } + } + return merged, nil +} + +// mergedPoolStorage merges pool.VMStorage over the top-level base, mirroring how the operator +// resolves a pool's own vmstorage when reconciling it. A pool without its own override falls +// back to the base as-is. +func (cr *VMCluster) mergedPoolStorage(pool *VMClusterPool) (*VMStorage, error) { + if pool.VMStorage == nil { + return cr.Spec.VMStorage, nil + } + merged := pool.VMStorage.DeepCopy() + if cr.Spec.VMStorage != nil { + if err := MergeDeep(merged, cr.Spec.VMStorage, true); err != nil { + return nil, fmt.Errorf("pool %q vmstorage merge: %w", pool.Name, err) + } + } + return merged, nil +} + +// AsURL builds the Service URL for the given cluster component. poolName selects a specific +// entry from spec.pools instead of the shared (non-pool) component; it's only meaningful for +// ClusterComponentInsert and ClusterComponentStorage, since vmselect is never pool-scoped. +// It errors instead of silently returning a URL for a Service the operator wouldn't actually +// create - e.g. the shared vminsert once any pool defines its own, or the top-level vmstorage +// once any pool is defined at all. +func (cr *VMCluster) AsURL(kind ClusterComponent, poolName string, isExtra bool) (string, error) { var defaultPort string var svcSpec *AdditionalServiceSpec var extraArgs map[string]string + svcName := cr.PrefixedName(kind) switch kind { case ClusterComponentSelect: + if poolName != "" { + return "", fmt.Errorf("vmcluster %q: pool %q is not applicable to vmselect, since vmselect is never pool-scoped", cr.Name, poolName) + } if cr.Spec.VMSelect == nil { - return "" + return "", fmt.Errorf("vmcluster %q has no spec.vmSelect configured", cr.Name) } defaultPort = "8481" if cr.Spec.VMSelect.Port != "" { @@ -1068,30 +1214,63 @@ func (cr *VMCluster) AsURL(kind ClusterComponent, isExtra bool) string { svcSpec = cr.Spec.VMSelect.ServiceSpec extraArgs = cr.Spec.VMSelect.ExtraArgs case ClusterComponentInsert: - if cr.Spec.VMInsert == nil { - return "" - } defaultPort = "8480" - if cr.Spec.VMInsert.Port != "" { - defaultPort = cr.Spec.VMInsert.Port + vmi := cr.Spec.VMInsert + if poolName != "" { + pool, ok := cr.findPool(poolName) + if !ok { + return "", fmt.Errorf("vmcluster %q has no pool named %q", cr.Name, poolName) + } + if pool.VMInsert != nil { + merged, err := cr.mergedPoolInsert(pool) + if err != nil { + return "", err + } + vmi = merged + svcName = cr.PoolPrefixedName(kind, poolName) + } + // pool has no dedicated vminsert: falls through to the shared top-level one below. + } else if hasAnyPoolInsert(cr.Spec.Pools) { + return "", fmt.Errorf("vmcluster %q has per-pool vminsert(s); target a specific pool instead of the shared vminsert", cr.Name) } - svcSpec = cr.Spec.VMInsert.ServiceSpec - extraArgs = cr.Spec.VMInsert.ExtraArgs - case ClusterComponentStorage: - if cr.Spec.VMStorage == nil { - return "" + if vmi == nil { + return "", fmt.Errorf("vmcluster %q has no shared spec.vmInsert configured", cr.Name) + } + if vmi.Port != "" { + defaultPort = vmi.Port } + svcSpec = vmi.ServiceSpec + extraArgs = vmi.ExtraArgs + case ClusterComponentStorage: defaultPort = "8482" - if cr.Spec.VMStorage.Port != "" { - defaultPort = cr.Spec.VMStorage.Port + vms := cr.Spec.VMStorage + if poolName != "" { + pool, ok := cr.findPool(poolName) + if !ok { + return "", fmt.Errorf("vmcluster %q has no pool named %q", cr.Name, poolName) + } + merged, err := cr.mergedPoolStorage(pool) + if err != nil { + return "", err + } + vms = merged + svcName = cr.PoolPrefixedName(kind, poolName) + } else if len(cr.Spec.Pools) > 0 { + return "", fmt.Errorf("vmcluster %q defines pools; target a specific pool instead of the top-level vmstorage", cr.Name) } - svcSpec = cr.Spec.VMStorage.ServiceSpec - extraArgs = cr.Spec.VMStorage.ExtraArgs + if vms == nil { + return "", fmt.Errorf("vmcluster %q has no shared spec.vmStorage configured", cr.Name) + } + if vms.Port != "" { + defaultPort = vms.Port + } + svcSpec = vms.ServiceSpec + extraArgs = vms.ExtraArgs default: panic("BUG unsupported cluster kind=" + string(kind)) } - svcName, port := ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + resolvedName, port := ResolveServiceURL(svcName, defaultPort, "http", svcSpec, isExtra) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(extraArgs), resolvedName, cr.Namespace, port), nil } func (cr *VMSelect) ProbePath() string { @@ -1231,3 +1410,40 @@ func (cr *VMAuthLoadBalancerSpec) UseTLS() bool { func (cr *VMAuthLoadBalancerSpec) GetMetricsPath() string { return BuildPathWithPrefixFlag(cr.ExtraArgs, metricsPath) } + +// maxVMClusterPoolNameLength must match the kubebuilder MaxLength on VMClusterPool.Name below. +const maxVMClusterPoolNameLength = 16 + +// VMClusterPool defines a named group of vmstorage (and optionally vminsert) components +// within a VMCluster. Each pool gets its own StatefulSet and headless Service. +// +k8s:openapi-gen=true +type VMClusterPool struct { + // Name is the unique identifier for this pool within the cluster. + // Used as a suffix for generated resource names (e.g. vmstorage--) and as a + // storage group name in vmselect. Kept short since the cluster name itself isn't length-limited, + // and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit. + // Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. + // +kubebuilder:validation:Pattern:="^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" + // +kubebuilder:validation:MaxLength=16 + Name string `json:"name"` + // VMStorage defines pool-specific vmstorage configuration. + // Each field overrides the corresponding field in the top-level vmstorage spec. + // Fields absent here inherit from the top-level vmstorage. + // RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. + // +optional + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + VMStorage *VMStorage `json:"vmstorage,omitempty"` + // VMInsert defines a dedicated vminsert for this pool. + // Each field overrides the corresponding field in the top-level vminsert spec. + // When nil, the top-level shared vminsert writes to this pool's storage nodes as well. + // +optional + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + VMInsert *VMInsert `json:"vminsert,omitempty"` +} + +// PoolPrefixedName returns the Kubernetes resource name for the given component in a pool. +func (cr *VMCluster) PoolPrefixedName(kind ClusterComponent, poolName string) string { + return ClusterPrefixedName(kind, cr.Name, "vm", false) + "-" + poolName +} diff --git a/api/operator/v1beta1/vmcluster_types_test.go b/api/operator/v1beta1/vmcluster_types_test.go index 9638349c9f..069c391f26 100644 --- a/api/operator/v1beta1/vmcluster_types_test.go +++ b/api/operator/v1beta1/vmcluster_types_test.go @@ -392,6 +392,30 @@ func TestVMCluster_Validate(t *testing.T) { cr.Spec.VMSelect.ExtraStorageNodes = []VMStorageNode{{Addr: managedAddr}} assert.Error(t, cr.Validate()) } + + // pools: no pool defines vminsert — shared top-level vminsert covers all of them, valid + f(VMClusterSpec{ + Pools: []VMClusterPool{ + {Name: "hot"}, + {Name: "cold"}, + }, + }, false) + + // pools: every pool defines its own vminsert — valid + f(VMClusterSpec{ + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + }, + }, false) + + // pools: only some pools define vminsert — the rest would have no ingestion path + f(VMClusterSpec{ + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold"}, + }, + }, true) } func TestVMCluster_PrefixedName(t *testing.T) { @@ -412,3 +436,66 @@ func TestVMCluster_PrefixedName(t *testing.T) { f("myapp", true, ClusterComponentInsert, "myapp-vminsert") f("myapp", true, ClusterComponentStorage, "myapp-vmstorage") } + +func TestVMCluster_AsURL(t *testing.T) { + newCR := func() *VMCluster { + return &VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-1", Namespace: "default"}, + Spec: VMClusterSpec{ + VMSelect: &VMSelect{}, + VMInsert: &VMInsert{}, + VMStorage: &VMStorage{}, + }, + } + } + fOK := func(cr *VMCluster, kind ClusterComponent, poolName string, wantHostPrefix string) { + t.Helper() + url, err := cr.AsURL(kind, poolName, false) + assert.NoError(t, err) + assert.Contains(t, url, wantHostPrefix) + } + fErr := func(cr *VMCluster, kind ClusterComponent, poolName string) { + t.Helper() + _, err := cr.AsURL(kind, poolName, false) + assert.Error(t, err) + } + + // no pools: shared components resolve normally. + cr := newCR() + fOK(cr, ClusterComponentSelect, "", "vmselect-cluster-1.") + fOK(cr, ClusterComponentInsert, "", "vminsert-cluster-1.") + fOK(cr, ClusterComponentStorage, "", "vmstorage-cluster-1.") + + // no pools: unset component errors instead of returning an empty/dangling URL. + cr = newCR() + cr.Spec.VMInsert = nil + fErr(cr, ClusterComponentInsert, "") + + // select is never pool-scoped. + cr = newCR() + fErr(cr, ClusterComponentSelect, "hot") + + // pools defined, none with a dedicated insert: shared insert still resolves for poolName="". + cr = newCR() + cr.Spec.Pools = []VMClusterPool{{Name: "hot"}, {Name: "cold"}} + fOK(cr, ClusterComponentInsert, "", "vminsert-cluster-1.") + // ...and also resolves when a pool without its own insert is named explicitly. + fOK(cr, ClusterComponentInsert, "hot", "vminsert-cluster-1.") + // top-level storage no longer exists once pools are defined. + fErr(cr, ClusterComponentStorage, "") + // but a named pool's own storage does. + fOK(cr, ClusterComponentStorage, "hot", "vmstorage-cluster-1-hot.") + // referencing an unknown pool is an error. + fErr(cr, ClusterComponentInsert, "unknown") + fErr(cr, ClusterComponentStorage, "unknown") + + // pools defined, every pool has its own dedicated insert: shared insert is ambiguous. + cr = newCR() + cr.Spec.Pools = []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + } + fErr(cr, ClusterComponentInsert, "") + fOK(cr, ClusterComponentInsert, "hot", "vminsert-cluster-1-hot.") + fOK(cr, ClusterComponentInsert, "cold", "vminsert-cluster-1-cold.") +} diff --git a/api/operator/v1beta1/vmextra_types.go b/api/operator/v1beta1/vmextra_types.go index c8ba6a6096..e18601f763 100644 --- a/api/operator/v1beta1/vmextra_types.go +++ b/api/operator/v1beta1/vmextra_types.go @@ -1841,3 +1841,59 @@ func (bs *BytesString) String() string { } return string(*bs) } + +func mergeMapsRecursive(baseMap, overrideMap map[string]any) { + if len(overrideMap) == 0 { + return + } + for key, overrideValue := range overrideMap { + if baseVal, ok := baseMap[key]; ok { + if baseMapNested, isBaseMap := baseVal.(map[string]any); isBaseMap { + if overrideMapNested, isOverrideMap := overrideValue.(map[string]any); isOverrideMap { + mergeMapsRecursive(baseMapNested, overrideMapNested) + continue + } + } + } + baseMap[key] = overrideValue + } +} + +// MergeDeep merges an override object into a base one using JSON round-trip. +// Fields present in the override overwrite corresponding fields in the base. +// When reverse is true the roles are swapped: base fills absent fields in override +// (useful when the override should win and the base provides defaults). +func MergeDeep[T comparable](base, override T, reverse bool) error { + var zero T + if override == zero { + return nil + } + baseJSON, err := json.Marshal(base) + if err != nil { + return fmt.Errorf("failed to marshal base spec: %w", err) + } + overrideJSON, err := json.Marshal(override) + if err != nil { + return fmt.Errorf("failed to marshal override spec: %w", err) + } + var baseMap map[string]any + if err := json.Unmarshal(baseJSON, &baseMap); err != nil { + return fmt.Errorf("failed to unmarshal base spec to map: %w", err) + } + var overrideMap map[string]any + if err := json.Unmarshal(overrideJSON, &overrideMap); err != nil { + return fmt.Errorf("failed to unmarshal override spec to map: %w", err) + } + if reverse { + baseMap, overrideMap = overrideMap, baseMap + } + mergeMapsRecursive(baseMap, overrideMap) + mergedJSON, err := json.Marshal(baseMap) + if err != nil { + return fmt.Errorf("failed to marshal merged spec map: %w", err) + } + if err := json.Unmarshal(mergedJSON, base); err != nil { + return fmt.Errorf("failed to unmarshal merged spec JSON: %w", err) + } + return nil +} diff --git a/api/operator/v1beta1/vmextra_types_test.go b/api/operator/v1beta1/vmextra_types_test.go index c90ee705b6..7c4b2a1dae 100644 --- a/api/operator/v1beta1/vmextra_types_test.go +++ b/api/operator/v1beta1/vmextra_types_test.go @@ -402,3 +402,143 @@ func TestImage_Reference(t *testing.T) { f(Image{Repository: "victoriametrics/vmsingle", Tag: ""}, "victoriametrics/vmsingle:") } + +func TestDeepMerge(t *testing.T) { + type opts struct { + override *VMClusterSpec + validate func(base, merged *VMClusterSpec, err error) + } + f := func(oss ...opts) { + t.Helper() + base := &VMClusterSpec{ + ClusterVersion: "v1.0.0", + ServiceAccountName: "base", + RetentionPeriod: "30d", + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + ExtraArgs: map[string]string{"keep": "x", "override": "old"}, + }, + }, + VMInsert: &VMInsert{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + ExtraArgs: map[string]string{"insert-arg": "1"}, + }, + }, + } + merged := base.DeepCopy() + for _, o := range oss { + o.validate(base, merged, MergeDeep(merged, o.override, false)) + } + } + + // with extra args override + f(opts{ + override: &VMClusterSpec{ + ClusterVersion: "v1.2.3", + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(3)), + ExtraArgs: map[string]string{"override": "new", "add": "y"}, + }, + }, + ServiceAccountName: "zone-sa", + }, + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v1.2.3", merged.ClusterVersion) + assert.Equal(t, "zone-sa", merged.ServiceAccountName) + if !assert.NotNil(t, merged.VMSelect) || !assert.NotNil(t, merged.VMSelect.ReplicaCount) { + return + } + assert.Equal(t, int32(3), *merged.VMSelect.ReplicaCount) + assert.Equal(t, "x", merged.VMSelect.ExtraArgs["keep"]) + assert.Equal(t, "new", merged.VMSelect.ExtraArgs["override"]) + assert.Equal(t, "y", merged.VMSelect.ExtraArgs["add"]) + if !assert.NotNil(t, merged.VMInsert) || !assert.NotNil(t, merged.VMInsert.ReplicaCount) { + return + } + assert.Equal(t, int32(1), *merged.VMInsert.ReplicaCount) + assert.Equal(t, "1", merged.VMInsert.ExtraArgs["insert-arg"]) + }, + }) + + // with nil override spec + f(opts{ + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, base, merged) + }, + }) + + // with empty override spec + f(opts{ + override: &VMClusterSpec{}, + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, base, merged) + }, + }) + + // with override spec that modifies top-level fields + f(opts{ + override: &VMClusterSpec{ + ClusterVersion: "v2.0.0", + ServiceAccountName: "global-sa", + }, + validate: func(_, merged *VMClusterSpec, err error) { + assert.Equal(t, "v2.0.0", merged.ClusterVersion) + assert.Equal(t, "global-sa", merged.ServiceAccountName) + assert.Equal(t, "30d", merged.RetentionPeriod) + }, + }) + + // multiple overrides applied in sequence + f(opts{ + override: &VMClusterSpec{ + ClusterVersion: "v2.0.0", + ServiceAccountName: "global-sa", + }, + validate: func(_, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v2.0.0", merged.ClusterVersion) + assert.Equal(t, "global-sa", merged.ServiceAccountName) + assert.Equal(t, "30d", merged.RetentionPeriod) + }, + }, opts{ + override: &VMClusterSpec{ + RetentionPeriod: "10d", + ClusterVersion: "v3.0.0", + }, + validate: func(_, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v3.0.0", merged.ClusterVersion) + assert.Equal(t, "global-sa", merged.ServiceAccountName) + assert.Equal(t, "10d", merged.RetentionPeriod) + }, + }) +} + +func TestMergeMapsRecursive(t *testing.T) { + base := map[string]any{ + "a": map[string]any{ + "b": "keep", + "c": "override-me", + }, + "d": "root-keep", + } + override := map[string]any{ + "a": map[string]any{ + "c": "new", + "z": "added", + }, + "e": "root-added", + } + mergeMapsRecursive(base, override) + assert.Equal(t, "keep", base["a"].(map[string]any)["b"]) + assert.Equal(t, "new", base["a"].(map[string]any)["c"]) + assert.Equal(t, "added", base["a"].(map[string]any)["z"]) + assert.Equal(t, "root-keep", base["d"]) + assert.Equal(t, "root-added", base["e"]) +} diff --git a/api/operator/v1beta1/vmsingle_types.go b/api/operator/v1beta1/vmsingle_types.go index 85b3373aba..c55f2f88f9 100644 --- a/api/operator/v1beta1/vmsingle_types.go +++ b/api/operator/v1beta1/vmsingle_types.go @@ -376,16 +376,20 @@ func (cr *VMSingle) GetRBACName() string { } func (cr *VMSingle) GetRemoteWriteURL() string { - return cr.AsURL(false) + BuildPathWithPrefixFlag(cr.Spec.ExtraArgs, "/api/v1/write") + url, err := cr.AsURL(false) + if err != nil { + return "" + } + return url + BuildPathWithPrefixFlag(cr.Spec.ExtraArgs, "/api/v1/write") } -func (cr *VMSingle) AsURL(isExtra bool) string { +func (cr *VMSingle) AsURL(isExtra bool) (string, error) { specPort := cr.Spec.Port if specPort == "" { specPort = "8428" } svcName, port := ResolveServiceURL(cr.PrefixedName(), specPort, "http", cr.Spec.ServiceSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(cr.Spec.ExtraArgs), svcName, cr.Namespace, port), nil } // LastSpecUpdated compares spec with last applied spec stored, replaces old spec and returns true if it's updated diff --git a/api/operator/v1beta1/vmuser_types.go b/api/operator/v1beta1/vmuser_types.go index 0a0ef5ad56..8ddf0b5131 100644 --- a/api/operator/v1beta1/vmuser_types.go +++ b/api/operator/v1beta1/vmuser_types.go @@ -231,6 +231,10 @@ type NamespacedName struct { // (created via spec.serviceSpec) over the default service when building the target URL. // +optional UseExtraService bool `json:"useExtraService,omitempty"` + // Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + // (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + // +optional + Pool string `json:"pool,omitempty"` } // CRDRef describe CRD target reference. @@ -247,7 +251,7 @@ type CRDRef struct { // AsKey returns unique key for object func (cr *CRDRef) AsKey(nsn NamespacedName) string { - return fmt.Sprintf("%s/%s/%s", cr.Kind, nsn.Namespace, nsn.Name) + return fmt.Sprintf("%s/%s/%s/%s", cr.Kind, nsn.Namespace, nsn.Name, nsn.Pool) } // StaticRef - user-defined routing host address. diff --git a/api/operator/v1beta1/zz_generated.deepcopy.go b/api/operator/v1beta1/zz_generated.deepcopy.go index 415b12c6a2..ea5d0375cd 100644 --- a/api/operator/v1beta1/zz_generated.deepcopy.go +++ b/api/operator/v1beta1/zz_generated.deepcopy.go @@ -6169,6 +6169,31 @@ func (in *VMClusterList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMClusterPool) DeepCopyInto(out *VMClusterPool) { + *out = *in + if in.VMStorage != nil { + in, out := &in.VMStorage, &out.VMStorage + *out = new(VMStorage) + (*in).DeepCopyInto(*out) + } + if in.VMInsert != nil { + in, out := &in.VMInsert, &out.VMInsert + *out = new(VMInsert) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterPool. +func (in *VMClusterPool) DeepCopy() *VMClusterPool { + if in == nil { + return nil + } + out := new(VMClusterPool) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMClusterSpec) DeepCopyInto(out *VMClusterSpec) { *out = *in @@ -6223,6 +6248,13 @@ func (in *VMClusterSpec) DeepCopyInto(out *VMClusterSpec) { *out = new(VMClusterDiscovery) **out = **in } + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]VMClusterPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterSpec. diff --git a/config/crd/overlay/crd.descriptionless.yaml b/config/crd/overlay/crd.descriptionless.yaml index 934e98b3f0..3115969a48 100644 --- a/config/crd/overlay/crd.descriptionless.yaml +++ b/config/crd/overlay/crd.descriptionless.yaml @@ -5631,6 +5631,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -5638,6 +5640,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -8727,6 +8731,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -8734,6 +8740,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -30374,6 +30382,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -30381,6 +30391,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -31657,6 +31669,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -31664,6 +31678,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -32329,6 +32345,24 @@ spec: type: object paused: type: boolean + pools: + items: + properties: + name: + maxLength: 16 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + vminsert: + x-kubernetes-preserve-unknown-fields: true + vmstorage: + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map replicationFactor: format: int32 type: integer @@ -34851,6 +34885,9 @@ spec: - retention type: object type: array + retentionPeriod: + pattern: ^[0-9]+(h|d|w|y)?$ + type: string revisionHistoryLimitCount: format: int32 type: integer @@ -36771,6 +36808,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -36778,6 +36817,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -39867,6 +39908,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -39874,6 +39917,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -62923,6 +62968,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -62930,6 +62977,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index 6a78f92791..252a763257 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -11280,6 +11280,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -11290,6 +11295,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -17882,6 +17892,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -17892,6 +17907,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -62563,6 +62583,11 @@ spec: namespace: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -62573,6 +62598,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -65173,6 +65203,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -65183,6 +65218,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -66401,6 +66441,50 @@ spec: Paused If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. type: boolean + pools: + description: |- + Pools defines named groups of vmstorage (and optionally vminsert) components. + Each pool gets its own StatefulSet and headless Service named --. + Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field. + vmselect queries all pools using the pool name as a storage group name (-storageNode=/). + When pools are defined the top-level vmstorage is not deployed; pools replace it entirely. + The top-level vminsert is deployed as a shared insert group across all pools only when no pool + defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. + items: + description: |- + VMClusterPool defines a named group of vmstorage (and optionally vminsert) components + within a VMCluster. Each pool gets its own StatefulSet and headless Service. + properties: + name: + description: |- + Name is the unique identifier for this pool within the cluster. + Used as a suffix for generated resource names (e.g. vmstorage--) and as a + storage group name in vmselect. Kept short since the cluster name itself isn't length-limited, + and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit. + Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. + maxLength: 16 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + vminsert: + description: |- + VMInsert defines a dedicated vminsert for this pool. + Each field overrides the corresponding field in the top-level vminsert spec. + When nil, the top-level shared vminsert writes to this pool's storage nodes as well. + x-kubernetes-preserve-unknown-fields: true + vmstorage: + description: |- + VMStorage defines pool-specific vmstorage configuration. + Each field overrides the corresponding field in the top-level vmstorage spec. + Fields absent here inherit from the top-level vmstorage. + RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map replicationFactor: description: |- ReplicationFactor defines how many copies of data make among @@ -71472,6 +71556,12 @@ spec: - retention type: object type: array + retentionPeriod: + description: |- + RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance. + Useful when using Pools to implement multi-retention setups. + pattern: ^[0-9]+(h|d|w|y)?$ + type: string revisionHistoryLimitCount: description: |- The number of old ReplicaSets to retain to allow rollback in deployment or @@ -75363,6 +75453,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -75373,6 +75468,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -81965,6 +82065,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -81975,6 +82080,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -127985,6 +128095,11 @@ spec: namespace: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -127995,6 +128110,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 61786ceba8..cc01ec3812 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,7 +17,7 @@ aliases: * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VL apps to [v1.52.0](https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/v1.52.0). * FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): add `networkPolicy` field to all supported CRDs (`VMSingle`, `VMAgent`, `VMAlert`, `VMAlertmanager`, `VMAuth`, `VLSingle`, `VLAgent`, `VTSingle`, `VMAnomaly`, and all cluster sub-components). When set, the operator creates and manages a `NetworkPolicy` resource that restricts ingress/egress to the component's pods. See [#2977](https://github.com/VictoriaMetrics/helm-charts/issues/2977). -* BUGFIX: [vmoperator](https://docs.victoriametrics.com/operator/): grant the operator's `ClusterRole` permissions on `networking.k8s.io/networkpolicies`. The `networkPolicy` feature above was shipped without this grant, so the operator would fail with a `Forbidden` error creating/updating/deleting `NetworkPolicy` objects on a real cluster. +* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/): add `spec.pools` support for multi-retention and tiered-storage setups. Each named pool runs isolated `vmstorage` nodes and optionally a dedicated `vminsert`, all queried transparently by `vmselect` as named storage groups. `VMUser`'s `targetRef.pool` field lets a user route to a specific pool's `vminsert`/`vmstorage` instead of the cluster's shared endpoint; referencing an ambiguous or nonexistent endpoint (e.g. the shared `vminsert` once every pool has its own) now fails config generation instead of silently producing a broken or empty backend URL. See [#741](https://github.com/VictoriaMetrics/operator/issues/741). * FEATURE: [vlagent](https://docs.victoriametrics.com/operator/resources/vlagent/): add `basicAuth` field support to `remoteWrite` entries. See [#2371](https://github.com/VictoriaMetrics/operator/issues/2371). * FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/): support VMCluster extraStorageNodes for vmselect component. * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): add `defaultVMAccessClaim` field to `spec.jwt`, mapped to vmauth's `jwt.default_vm_access_claim`. It lets a `VMUser` accept JWTs that don't carry a `vm_access` claim, matching `vmauth` v1.147.0+ behavior. See [#2375](https://github.com/VictoriaMetrics/operator/issues/2375). @@ -29,7 +29,6 @@ aliases: * BUGFIX: [vmoperator](https://docs.victoriametrics.com/operator/): return an error instead of panicking when a `Labels` map value is malformed (missing the `=` separator) during config parsing. * BUGFIX: [vmalert](https://docs.victoriametrics.com/operator/resources/vmalert/): when no notifiers are configured, ignore alerting rules from selected VMRules instead of failing reconciliation; recording rules in the same group are kept and still reconciled. See [#2388](https://github.com/VictoriaMetrics/operator/issues/2388). * BUGFIX: [vmalertmanager](https://docs.victoriametrics.com/operator/resources/vmalertmanager/): default pod `securityContext.fsGroup` to `65534` when persistent storage is configured and neither `useStrictSecurity` nor a user `securityContext` is set. Without it the alertmanager process cannot write notification log and silences to a freshly provisioned volume, so silences are silently lost on pod restart. User-provided `securityContext` and `useStrictSecurity` keep their previous behaviour. - * BUGFIX: [vmoperator](https://docs.victoriametrics.com/operator/): allow pinning images by digest via the `image.tag` field of all operator CRs. When `tag` is a digest (e.g. `sha256:`) it is now joined to the repository with `@` instead of `:`, producing a valid `repository@sha256:` reference. Regular tags are unaffected. * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): fix `persistentVolume.storageClassName` being dropped during conversion due to a stale `storageClass` field name. See [#2389](https://github.com/VictoriaMetrics/operator/issues/2389). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): fix `securityContext.runAsNonRoot`, `runAsUser`, `runAsGroup`, `seccompProfile`, `appArmorProfile`, `seLinuxOptions`, and `windowsOptions` being silently dropped during conversion; these are now promoted to `spec.securityContext.podSecurityContext`, matching how the operator applies them to containers. See [#2391](https://github.com/VictoriaMetrics/operator/issues/2391). diff --git a/docs/api.md b/docs/api.md index ed573bd9e2..997567c0e9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1732,6 +1732,7 @@ Appears in: [TargetRef](#targetref) | name#
_string_ | _(Required)_
Name of the target Kubernetes object | | namespace#
_string_ | _(Required)_
Namespace of the target Kubernetes object | | objects#
_[NamespacedName](#namespacedname) array_ | _(Optional)_
Objects defines list of name/namespace pairs that define existing k8s object | +| pool#
_string_ | _(Optional)_
Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared
(non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. | | useExtraService#
_boolean_ | _(Optional)_
UseExtraService instructs the operator to prefer the CR's additional service
(created via spec.serviceSpec) over the default service when building the target URL. | #### Certs @@ -2853,6 +2854,7 @@ Appears in: [CRDRef](#crdref) | --- | --- | | name#
_string_ | _(Required)_
Name of the target Kubernetes object | | namespace#
_string_ | _(Required)_
Namespace of the target Kubernetes object | +| pool#
_string_ | _(Optional)_
Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared
(non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. | | useExtraService#
_boolean_ | _(Optional)_
UseExtraService instructs the operator to prefer the CR's additional service
(created via spec.serviceSpec) over the default service when building the target URL. | #### NomadSDConfig @@ -4654,6 +4656,19 @@ Appears in: [VMClusterSpec](#vmclusterspec), [VMInsert](#vminsert), [VMSelect](# | filter#
_string_ | _(Optional)_
Filter is an optional regexp filter applied to discovered vmstorage addresses.
Only addresses matching the filter are used; non-matching addresses are ignored. | | interval#
_string_ | _(Optional)_
Interval is the interval for refreshing the vmstorage node list resolved from DNS SRV records.
The minimum supported value is 1s.
Defaults to 2s if not set. | +#### VMClusterPool + +VMClusterPool defines a named group of vmstorage (and optionally vminsert) components +within a VMCluster. Each pool gets its own StatefulSet and headless Service. + +Appears in: [VMClusterSpec](#vmclusterspec) + +| Field | Description | +| --- | --- | +| name#
_string_ | _(Required)_
Name is the unique identifier for this pool within the cluster.
Used as a suffix for generated resource names (e.g. vmstorage--) and as a
storage group name in vmselect. Kept short since the cluster name itself isn't length-limited,
and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit.
Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. | +| vminsert#
_[VMInsert](#vminsert)_ | _(Optional)_
VMInsert defines a dedicated vminsert for this pool.
Each field overrides the corresponding field in the top-level vminsert spec.
When nil, the top-level shared vminsert writes to this pool's storage nodes as well. | +| vmstorage#
_[VMStorage](#vmstorage)_ | _(Optional)_
VMStorage defines pool-specific vmstorage configuration.
Each field overrides the corresponding field in the top-level vmstorage spec.
Fields absent here inherit from the top-level vmstorage.
RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. | + #### VMClusterSpec VMClusterSpec defines the desired state of VMCluster @@ -4670,6 +4685,7 @@ Appears in: [VMCluster](#vmcluster), [VMDistributedZoneCluster](#vmdistributedzo | license#
_[License](#license)_ | _(Optional)_
License allows to configure license key to be used for enterprise features.
Using license key is supported starting from VictoriaMetrics v1.94.0.
See [here](https://docs.victoriametrics.com/victoriametrics/enterprise/) | | managedMetadata#
_[ManagedObjectsMetadata](#managedobjectsmetadata)_ | _(Required)_
ManagedMetadata defines metadata that will be added to the all objects
created by operator for the given CustomResource | | paused#
_boolean_ | _(Optional)_
Paused If set to true all actions on the underlying managed objects are not
going to be performed, except for delete actions. | +| pools#
_[VMClusterPool](#vmclusterpool) array_ | _(Optional)_
Pools defines named groups of vmstorage (and optionally vminsert) components.
Each pool gets its own StatefulSet and headless Service named --.
Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field.
vmselect queries all pools using the pool name as a storage group name (-storageNode=/).
When pools are defined the top-level vmstorage is not deployed; pools replace it entirely.
The top-level vminsert is deployed as a shared insert group across all pools only when no pool
defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. | | replicationFactor#
_integer_ | _(Optional)_
ReplicationFactor defines how many copies of data make among
distinct storage nodes | | requestsLoadBalancer#
_[VMAuthLoadBalancer](#vmauthloadbalancer)_ | _(Required)_
RequestsLoadBalancer configures load-balancing for vminsert and vmselect requests.
It helps to evenly spread load across pods.
Usually it's not possible with Kubernetes TCP-based services.
See more [here](https://docs.victoriametrics.com/operator/resources/vmcluster/#requests-load-balancing) | | retentionPeriod#
_string_ | _(Optional)_
RetentionPeriod defines how long to retain stored metrics, specified as a duration (e.g., "1d", "1w", "1m").
Data with timestamps outside the RetentionPeriod is automatically deleted. The minimum allowed value is 1d, or 24h.
The default value is 1 (one month).
See [retention](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) docs for details. | @@ -4682,7 +4698,7 @@ Appears in: [VMCluster](#vmcluster), [VMDistributedZoneCluster](#vmdistributedzo #### VMInsert -Appears in: [VMClusterSpec](#vmclusterspec) +Appears in: [VMClusterPool](#vmclusterpool), [VMClusterSpec](#vmclusterspec) | Field | Description | | --- | --- | @@ -5307,7 +5323,7 @@ Appears in: [VMStaticScrape](#vmstaticscrape) #### VMStorage -Appears in: [VMClusterSpec](#vmclusterspec) +Appears in: [VMClusterPool](#vmclusterpool), [VMClusterSpec](#vmclusterspec) | Field | Description | | --- | --- | @@ -5351,6 +5367,7 @@ Appears in: [VMClusterSpec](#vmclusterspec) | replicaCount#
_integer_ | _(Optional)_
ReplicaCount is the expected size of the Application. | | resources#
_[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#resourcerequirements-v1-core)_ | _(Optional)_
Resources container resource request and limits, https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
if not defined default resources from operator config will be used | | retentionFilters#
_[RetentionFiltersConfig](#retentionfiltersconfig)_ | _(Optional)_
RetentionFilters defines per-series retention filters for vmstorage.
Requires enterprise license. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters | +| retentionPeriod#
_string_ | _(Optional)_
RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance.
Useful when using Pools to implement multi-retention setups. | | revisionHistoryLimitCount#
_integer_ | _(Optional)_
The number of old ReplicaSets to retain to allow rollback in deployment or
maximum number of revisions that will be maintained in the Deployment revision history.
Has no effect at StatefulSets
Defaults to 10. | | rollingUpdateStrategy#
_[StatefulSetUpdateStrategyType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#statefulsetupdatestrategytype-v1-apps)_ | _(Optional)_
RollingUpdateStrategy defines strategy for application updates
Default is OnDelete, in this case operator handles update process
Can be changed for RollingUpdate | | rollingUpdateStrategyBehavior#
_[StatefulSetUpdateStrategyBehavior](#statefulsetupdatestrategybehavior)_ | _(Optional)_
RollingUpdateStrategyBehavior defines customized behavior for rolling updates.
It applies if the RollingUpdateStrategy is set to OnDelete, which is the default. | diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md index 9c6e219d9f..729b9ed016 100644 --- a/docs/resources/vmcluster.md +++ b/docs/resources/vmcluster.md @@ -863,6 +863,122 @@ spec: enabled: false ``` +## Storage pools + +`spec.pools` allows you to run multiple isolated groups of `vmstorage` nodes within a single `VMCluster`. +Each pool has its own name, StatefulSet, headless Service, and optional PodDisruptionBudget / HPA / VPA. +`vmselect` queries all pools simultaneously and merges results transparently, so existing queries require no changes. + +The primary use case is **multi-retention**: assign a different `retentionPeriod` to each pool so that some metrics expire sooner than others, without deploying separate clusters. + +When `spec.pools` is defined the top-level `spec.vmstorage` is **not** deployed; pools replace it entirely. + +### Pool configuration and inheritance + +Each pool entry has a `name` and an optional `vmstorage` override: + +| Field | Description | +|-------|-------------| +| `name` | Unique identifier within the cluster. Used as a resource name suffix and as the storage group name in `vmselect`. Must match `^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`, max 32 chars. | +| `vmstorage` | Pool-specific `vmstorage` configuration. Each field overrides the corresponding field in the top-level `spec.vmstorage`; absent fields inherit from the top-level. | +| `vminsert` | Optional dedicated `vminsert` for this pool. When nil, the shared top-level `vminsert` writes to this pool's storage nodes as well. | + +`vmstorage.retentionPeriod` inside a pool overrides `spec.retentionPeriod` for that pool, enabling per-pool retention without enterprise retention filters. + +### Resource naming + +Pool resources follow the pattern `vm--`: + +| Resource | Example | +|----------|---------| +| `vmstorage` StatefulSet | `vmstorage-mycluster-hot` | +| `vmstorage` headless Service | `vmstorage-mycluster-hot` | +| `vminsert` Deployment (dedicated) | `vminsert-mycluster-hot` | + +All pool pods carry an extra label `app.kubernetes.io/pool=` to keep per-pool selectors disjoint. + +### Multi-retention example + +Two pools with different retention periods sharing a single `vminsert` and `vmselect`: + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: example-pools +spec: + retentionPeriod: "30d" # default, used as fallback + vmselect: + replicaCount: 2 + vminsert: + replicaCount: 2 # shared across both pools + # top-level vmstorage sets shared defaults inherited by all pools + vmstorage: + replicaCount: 3 + storage: + volumeClaimTemplate: + spec: + resources: + requests: + storage: 50Gi + pools: + - name: hot + vmstorage: + retentionPeriod: "7d" + - name: cold + vmstorage: + replicaCount: 5 + retentionPeriod: "365d" + storage: + volumeClaimTemplate: + spec: + resources: + requests: + storage: 2Ti +``` + +This creates: +- `vmstorage-example-pools-hot` StatefulSet with 3 replicas (inherited) and 7-day retention. +- `vmstorage-example-pools-cold` StatefulSet with 5 replicas (overridden) and 365-day retention. +- One shared `vminsert-example-pools` Deployment writing to both pools. +- `vmselect` queries both pools and returns merged results. + +### Pool with a dedicated vminsert + +When a pool defines `vminsert`, it gets its own isolated insert path. +The shared top-level `vminsert` is **not** deployed when every pool has a dedicated one. +Use this when pools need different write-path configurations (e.g. different stream aggregation rules or resource limits): + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: example-dedicated-insert +spec: + retentionPeriod: "30d" + vmselect: + replicaCount: 2 + # top-level vminsert sets shared defaults inherited by pool-level inserts + vminsert: + replicaCount: 2 + resources: + limits: + cpu: "1" + memory: 512Mi + vmstorage: + replicaCount: 3 + pools: + - name: short + vmstorage: + retentionPeriod: "7d" + vminsert: + replicaCount: 4 # override: more replicas for high-throughput writes + - name: long + vmstorage: + retentionPeriod: "2y" + vminsert: {} # dedicated insert inheriting all top-level vminsert defaults +``` + ## Examples ### Minimal example without persistence diff --git a/internal/controller/operator/factory/build/build.go b/internal/controller/operator/factory/build/build.go index 6ccc17ee03..40506fc47c 100644 --- a/internal/controller/operator/factory/build/build.go +++ b/internal/controller/operator/factory/build/build.go @@ -3,7 +3,6 @@ package build import ( "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "path" @@ -318,68 +317,3 @@ func GunzipConfig(data []byte) ([]byte, error) { defer gr.Close() return io.ReadAll(gr) } - -// mergeMapsRecursive deeply merges overrideMap into baseMap. -// It handles nested maps (which correspond to nested structs after JSON unmarshal). -// Values from overrideMap overwrite values in baseMap. -func mergeMapsRecursive(baseMap, overrideMap map[string]any) { - if len(overrideMap) == 0 { - return - } - for key, overrideValue := range overrideMap { - if baseVal, ok := baseMap[key]; ok { - if baseMapNested, isBaseMap := baseVal.(map[string]any); isBaseMap { - if overrideMapNested, isOverrideMap := overrideValue.(map[string]any); isOverrideMap { - // Both are nested maps, recurse - mergeMapsRecursive(baseMapNested, overrideMapNested) - continue - } - } - } - baseMap[key] = overrideValue - } -} - -// MergeDeep merges an override object into a base one. -// Fields present in the override will overwrite corresponding fields in the base. -func MergeDeep[T comparable](base, override T, reverse bool) error { - var zero T - if override == zero { - return nil - } - - baseJSON, err := json.Marshal(base) - if err != nil { - return fmt.Errorf("failed to marshal base spec: %w", err) - } - overrideJSON, err := json.Marshal(override) - if err != nil { - return fmt.Errorf("failed to marshal override spec: %w", err) - } - - var baseMap map[string]any - if err := json.Unmarshal(baseJSON, &baseMap); err != nil { - return fmt.Errorf("failed to unmarshal base spec to map: %w", err) - } - var overrideMap map[string]any - if err := json.Unmarshal(overrideJSON, &overrideMap); err != nil { - return fmt.Errorf("failed to unmarshal override spec to map: %w", err) - } - - // Perform a deep merge: fields from overrideMap recursively overwrite corresponding fields in baseMap. - // If an override value is explicitly nil, it signifies the removal or nullification of that field. - if reverse { - baseMap, overrideMap = overrideMap, baseMap - } - mergeMapsRecursive(baseMap, overrideMap) - mergedSpecJSON, err := json.Marshal(baseMap) - if err != nil { - return fmt.Errorf("failed to marshal merged spec map: %w", err) - } - - if err := json.Unmarshal(mergedSpecJSON, base); err != nil { - return fmt.Errorf("failed to unmarshal merged spec JSON: %w", err) - } - - return nil -} diff --git a/internal/controller/operator/factory/build/build_test.go b/internal/controller/operator/factory/build/build_test.go index 1241c7a7d2..ddee8e2ae5 100644 --- a/internal/controller/operator/factory/build/build_test.go +++ b/internal/controller/operator/factory/build/build_test.go @@ -188,150 +188,3 @@ func TestGzipGunzipConfig(t *testing.T) { // binary data f([]int32{1, 2, 3, 4}) } - -func TestMergeMapsRecursive(t *testing.T) { - base := map[string]any{ - "a": map[string]any{ - "b": "keep", - "c": "override-me", - }, - "d": "root-keep", - } - override := map[string]any{ - "a": map[string]any{ - "c": "new", - "z": "added", - }, - "e": "root-added", - } - // initial merge - mergeMapsRecursive(base, override) - assert.Equal(t, "keep", base["a"].(map[string]any)["b"]) - assert.Equal(t, "new", base["a"].(map[string]any)["c"]) - assert.Equal(t, "added", base["a"].(map[string]any)["z"]) - assert.Equal(t, "root-keep", base["d"]) - assert.Equal(t, "root-added", base["e"]) -} - -func TestDeepMerge(t *testing.T) { - type opts struct { - override *vmv1beta1.VMClusterSpec - validate func(base, merged *vmv1beta1.VMClusterSpec, err error) - } - f := func(oss ...opts) { - t.Helper() - base := &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v1.0.0", - ServiceAccountName: "base", - RetentionPeriod: "30d", - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - ExtraArgs: map[string]string{"keep": "x", "override": "old"}, - }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - ExtraArgs: map[string]string{"insert-arg": "1"}, - }, - }, - } - merged := base.DeepCopy() - for _, o := range oss { - o.validate(base, merged, MergeDeep(merged, o.override, false)) - } - } - - // with extra args override - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v1.2.3", - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(3)), - ExtraArgs: map[string]string{"override": "new", "add": "y"}, - }, - }, - ServiceAccountName: "zone-sa", - }, - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - - // top-level - assert.Equal(t, "v1.2.3", merged.ClusterVersion) - assert.Equal(t, "zone-sa", merged.ServiceAccountName) - - // nested merge - if !assert.NotNil(t, merged.VMSelect) || !assert.NotNil(t, merged.VMSelect.ReplicaCount) { - return - } - assert.Equal(t, int32(3), *merged.VMSelect.ReplicaCount) - assert.Equal(t, "x", merged.VMSelect.ExtraArgs["keep"]) - assert.Equal(t, "new", merged.VMSelect.ExtraArgs["override"]) - assert.Equal(t, "y", merged.VMSelect.ExtraArgs["add"]) - - // untouched subtree - if !assert.NotNil(t, merged.VMInsert) || !assert.NotNil(t, merged.VMInsert.ReplicaCount) { - return - } - assert.Equal(t, int32(1), *merged.VMInsert.ReplicaCount) - assert.Equal(t, "1", merged.VMInsert.ExtraArgs["insert-arg"]) - }, - }) - - // with nil override spec - f(opts{ - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, base, merged) - }, - }) - - // with empty override spec - f(opts{ - override: &vmv1beta1.VMClusterSpec{}, - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, base, merged) - }, - }) - - // with override spec that modifies top-level fields - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v2.0.0", - ServiceAccountName: "global-sa", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.Equal(t, "v2.0.0", merged.ClusterVersion) - assert.Equal(t, "global-sa", merged.ServiceAccountName) - assert.Equal(t, "30d", merged.RetentionPeriod) - }, - }) - - // multiple overrides - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v2.0.0", - ServiceAccountName: "global-sa", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, "v2.0.0", merged.ClusterVersion) - assert.Equal(t, "global-sa", merged.ServiceAccountName) - assert.Equal(t, "30d", merged.RetentionPeriod) - }, - }, opts{ - override: &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "10d", - ClusterVersion: "v3.0.0", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, "v3.0.0", merged.ClusterVersion) // Cluster-specific override should take precedence - assert.Equal(t, "global-sa", merged.ServiceAccountName) // From global override, unchanged by cluster override - assert.Equal(t, "10d", merged.RetentionPeriod) // From cluster override - }, - }) -} diff --git a/internal/controller/operator/factory/build/cluster.go b/internal/controller/operator/factory/build/cluster.go index 76e917979b..bd6c8a0637 100644 --- a/internal/controller/operator/factory/build/cluster.go +++ b/internal/controller/operator/factory/build/cluster.go @@ -25,11 +25,12 @@ type ChildBuilder struct { kind vmv1beta1.ClusterComponent finalLabels map[string]string selectorLabels map[string]string + prefixedName string } // PrefixedName implements build.svcBuilderArgs interface func (b *ChildBuilder) PrefixedName() string { - return b.ParentOpts.PrefixedName(b.kind) + return b.prefixedName } // FinalLabels implements build.svcBuilderArgs interface @@ -55,11 +56,30 @@ func (b *ChildBuilder) SetSelectorLabels(ls map[string]string) { b.selectorLabels = ls } +// poolLabelKey is the label added to all pool-scoped resources. +const poolLabelKey = "app.kubernetes.io/pool" + func NewChildBuilder(cr ParentOpts, kind vmv1beta1.ClusterComponent) *ChildBuilder { + return NewPoolBuilder(cr, kind, "") +} + +// NewPoolBuilder is like NewChildBuilder but scopes the builder to a named pool. +// PrefixedName appends "-", and both SelectorLabels and FinalLabels include +// the pool label so that multiple pools in the same namespace have non-overlapping selectors. +func NewPoolBuilder(cr ParentOpts, kind vmv1beta1.ClusterComponent, poolName string) *ChildBuilder { + selectorLabels := cr.SelectorLabels(kind) + finalLabels := cr.FinalLabels(kind) + name := cr.PrefixedName(kind) + if poolName != "" { + name += "-" + poolName + selectorLabels[poolLabelKey] = poolName + finalLabels[poolLabelKey] = poolName + } return &ChildBuilder{ ParentOpts: cr, kind: kind, - finalLabels: cr.FinalLabels(kind), - selectorLabels: cr.SelectorLabels(kind), + finalLabels: finalLabels, + selectorLabels: selectorLabels, + prefixedName: name, } } diff --git a/internal/controller/operator/factory/vmauth/vmusers_config.go b/internal/controller/operator/factory/vmauth/vmusers_config.go index b064b409ac..ee9f8fe0e7 100644 --- a/internal/controller/operator/factory/vmauth/vmusers_config.go +++ b/internal/controller/operator/factory/vmauth/vmusers_config.go @@ -53,6 +53,11 @@ func updateCRDObjURLs(ctx context.Context, rclient client.Client, crd *vmv1beta1 } crdObj.SetName(nsn.Name) crdObj.SetNamespace(nsn.Namespace) + if cw, ok := crdObj.(*clusterWithURL); ok { + cw.pool = nsn.Pool + } else if nsn.Pool != "" { + return fmt.Errorf("pool=%q is not supported for kind=%q", nsn.Pool, crd.Kind) + } url, err := getAsURLObject(ctx, rclient, crdObj, nsn.UseExtraService) if err != nil { if !build.IsNotFound(err) { @@ -79,7 +84,7 @@ func (pos *parsedObjects) buildConfig(ctx context.Context, rclient client.Client backends[ref.Name] = ref } else { if r, ok := backends[ref.Name]; ok { - if err := build.MergeDeep(ref, r, true); err != nil { + if err := vmv1beta1.MergeDeep(ref, r, true); err != nil { return fmt.Errorf("failed to merge target refs: %w", err) } } else { @@ -173,7 +178,7 @@ func createVMUserSecrets(ctx context.Context, rclient client.Client, secrets []* type objectWithURL interface { client.Object - AsURL(isExtra bool) string + AsURL(isExtra bool) (string, error) } func getAsURLObject(ctx context.Context, rclient client.Client, objT objectWithURL, isExtra bool) (string, error) { @@ -191,7 +196,7 @@ func getAsURLObject(ctx context.Context, rclient client.Client, objT objectWithU } return "", fmt.Errorf("cannot get object by given ref namespace=%q,name=%q: %w", obj.GetNamespace(), obj.GetName(), err) } - return objT.AsURL(isExtra), nil + return objT.AsURL(isExtra) } func (pos *parsedObjects) addAuthCredentialsBuildSecrets(ac *build.AssetsCache) (needToCreateSecrets []*corev1.Secret, needToUpdateSecrets []*corev1.Secret, resultErr error) { @@ -335,40 +340,52 @@ type unwrapObject interface { origin() client.Object } -var clusterComponentToURL = map[string]func(obj client.Object, isExtra bool) string{ - "vminsert": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) +// rejectPool wraps a pool-unaware cluster AsURL so its kind rejects a non-empty pool instead of +// silently ignoring it - only VMCluster currently supports spec.pools. +func rejectPool(kind string, f func(obj client.Object, isExtra bool) (string, error)) func(obj client.Object, poolName string, isExtra bool) (string, error) { + return func(obj client.Object, poolName string, isExtra bool) (string, error) { + if poolName != "" { + return "", fmt.Errorf("pool=%q is not supported for kind=%q", poolName, kind) + } + return f(obj, isExtra) + } +} + +var clusterComponentToURL = map[string]func(obj client.Object, poolName string, isExtra bool) (string, error){ + "vminsert": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentInsert, poolName, isExtra) }, - "vmselect": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) + "vmselect": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentSelect, poolName, isExtra) }, - "vmstorage": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) + "vmstorage": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentStorage, poolName, isExtra) }, - "vlinsert": func(obj client.Object, isExtra bool) string { + "vlinsert": rejectPool("VLCluster/vlinsert", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) - }, - "vlselect": func(obj client.Object, isExtra bool) string { + }), + "vlselect": rejectPool("VLCluster/vlselect", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) - }, - "vlstorage": func(obj client.Object, isExtra bool) string { + }), + "vlstorage": rejectPool("VLCluster/vlstorage", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) - }, - "vtinsert": func(obj client.Object, isExtra bool) string { + }), + "vtinsert": rejectPool("VTCluster/vtinsert", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) - }, - "vtselect": func(obj client.Object, isExtra bool) string { + }), + "vtselect": rejectPool("VTCluster/vtselect", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) - }, - "vtstorage": func(obj client.Object, isExtra bool) string { + }), + "vtstorage": rejectPool("VTCluster/vtstorage", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) - }, + }), } type clusterWithURL struct { client.Object originObj client.Object component string + pool string } func newClusterWithURL(component string) *clusterWithURL { @@ -383,7 +400,7 @@ func newClusterWithURL(component string) *clusterWithURL { default: panic(fmt.Sprintf("BUG: unexpected component name: %q", component)) } - return &clusterWithURL{clusterObj, clusterObj, component} + return &clusterWithURL{clusterObj, clusterObj, component, ""} } func (c *clusterWithURL) origin() client.Object { @@ -391,12 +408,12 @@ func (c *clusterWithURL) origin() client.Object { } // AsURL implements AsURL interface -func (c *clusterWithURL) AsURL(isExtra bool) string { +func (c *clusterWithURL) AsURL(isExtra bool) (string, error) { builder, ok := clusterComponentToURL[c.component] if !ok { panic(fmt.Sprintf("BUG: not expected component=%q for clusterWithURL object", c.component)) } - return builder(c.Object, isExtra) + return builder(c.Object, c.pool, isExtra) } // generateVMAuthConfig create VMAuth cfg for given Users. diff --git a/internal/controller/operator/factory/vmauth/vmusers_config_test.go b/internal/controller/operator/factory/vmauth/vmusers_config_test.go index eb011439c4..7aae4488b5 100644 --- a/internal/controller/operator/factory/vmauth/vmusers_config_test.go +++ b/internal/controller/operator/factory/vmauth/vmusers_config_test.go @@ -76,8 +76,8 @@ func Test_genUserCfg(t *testing.T) { }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -155,8 +155,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -203,8 +203,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -287,8 +287,8 @@ password: pass }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_map: - url_prefix: @@ -360,9 +360,9 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", - "VLogs/monitoring/db": "http://vlogs-b.monitoring.svc:8482", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", + "VLogs/monitoring/db/": "http://vlogs-b.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -409,8 +409,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_prefix: - http://vmagent-base.monitoring.svc:8429 @@ -442,8 +442,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_prefix: - http://vmagent-base.monitoring.svc:8429 @@ -740,11 +740,11 @@ password: pass }, }, objURLs: map[string]string{ - "VLAgent/monitoring/collector": "http://vlagent-base.monitoring.svc:9429", - "VLSingle/monitoring/db": "http://vlsingle-db.monitoring.svc:9428", - "VLCluster/vlinsert/monitoring/main-cluster": "http://vlinsert-main-cluster.monitoring.svc:9401", - "VLCluster/vlselect/monitoring/main-cluster": "http://vlselect-main-cluster.monitoring.svc:9401", - "VLCluster/vlstorage/monitoring/main-cluster": "http://vlstorage-main-cluster.monitoring.svc:9401", + "VLAgent/monitoring/collector/": "http://vlagent-base.monitoring.svc:9429", + "VLSingle/monitoring/db/": "http://vlsingle-db.monitoring.svc:9428", + "VLCluster/vlinsert/monitoring/main-cluster/": "http://vlinsert-main-cluster.monitoring.svc:9401", + "VLCluster/vlselect/monitoring/main-cluster/": "http://vlselect-main-cluster.monitoring.svc:9401", + "VLCluster/vlstorage/monitoring/main-cluster/": "http://vlstorage-main-cluster.monitoring.svc:9401", }, want: `url_map: - url_prefix: @@ -3350,6 +3350,54 @@ unauthorized_user: src_paths: - /insert/.* bearer_token: bearer +`, + }) + + // VMUser targeting a specific pool's vminsert on a VMCluster where every pool has its own + // dedicated insert (the shared top-level vminsert is ambiguous in that case). + f(opts{ + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{Name: "test-auth", Namespace: "default"}, + Spec: vmv1beta1.VMAuthSpec{SelectAllByDefault: true}, + }, + predefinedObjects: []runtime.Object{ + &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "pooled", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMSelect: &vmv1beta1.VMSelect{}, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMInsert: &vmv1beta1.VMInsert{}}, + {Name: "cold", VMInsert: &vmv1beta1.VMInsert{}}, + }, + }, + }, + &vmv1beta1.VMUser{ + ObjectMeta: metav1.ObjectMeta{Name: "user-1", Namespace: "default"}, + Spec: vmv1beta1.VMUserSpec{ + BearerToken: ptr.To("bearer"), + TargetRefs: []vmv1beta1.TargetRef{ + { + CRD: &vmv1beta1.CRDRef{ + Kind: "VMCluster/vminsert", + NamespacedName: vmv1beta1.NamespacedName{ + Name: "pooled", + Namespace: "default", + Pool: "hot", + }, + }, + Paths: []string{"/insert/.*"}, + }, + }, + }, + }, + }, + want: `users: +- url_map: + - url_prefix: + - http://vminsert-pooled-hot.default.svc:8480 + src_paths: + - /insert/.* + bearer_token: bearer `, }) } diff --git a/internal/controller/operator/factory/vmcluster/vmcluster.go b/internal/controller/operator/factory/vmcluster/vmcluster.go index 9c449b8834..6ac7db3d0d 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" vpav1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,13 +29,9 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" ) -// CreateOrUpdate reconciled cluster object with order -// first we check status of vmStorage and waiting for its readiness -// then vmSelect and wait for it readiness as well -// and last one is vmInsert -// we manually handle statefulsets rolling updates -// needed in update checked by revision status -// its controlled by k8s controller-manager +// CreateOrUpdate reconciles the cluster in order: vmstorage/vminsert per pool (waiting for +// readiness before moving on), then vmselect. StatefulSet rolling updates are handled manually, +// gated on revision status, since k8s controller-manager doesn't manage that for us here. func CreateOrUpdate(ctx context.Context, cr *vmv1beta1.VMCluster, rclient client.Client) error { if cr.Paused() { return nil @@ -82,85 +79,38 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1beta1.VMCluster, rclient client } } - if cr.Spec.VMStorage != nil { - if cr.Spec.VMStorage.PodDisruptionBudget != nil { - err := createOrUpdatePodDisruptionBudgetForVMStorage(ctx, rclient, cr, prevCR) - if err != nil { - return err - } - } - if cr.Spec.VMStorage.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMStorage(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if err := createOrUpdateVMStorage(ctx, rclient, cr, prevCR); err != nil { - return err - } + // base (non-pool) resources; createOrUpdatePool itself decides whether the shared + // top-level vmstorage/vminsert should still be created once pools are declared. + if err := createOrUpdatePool(ctx, rclient, cr, prevCR, nil, owner); err != nil { + return err + } - if err := createOrUpdateVMStorageService(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMStorageHPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMStorageVPA(ctx, rclient, cr, prevCR); err != nil { - return err + for _, pool := range cr.Spec.Pools { + if err := createOrUpdatePool(ctx, rclient, cr, prevCR, &pool, owner); err != nil { + return fmt.Errorf("pool %q: %w", pool.Name, err) } } if cr.Spec.VMSelect != nil { - if cr.Spec.VMSelect.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMSelect(ctx, rclient, cr, prevCR); err != nil { - return err - } + if err := createOrUpdatePodDisruptionBudgetForVMSelect(ctx, rclient, cr, prevCR); err != nil { + return err } - if cr.Spec.VMSelect.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMSelect(ctx, rclient, cr, prevCR); err != nil { - return err - } + if err := createOrUpdateNetworkPolicyForVMSelect(ctx, rclient, cr, prevCR); err != nil { + return err } if err := createOrUpdateVMSelect(ctx, rclient, cr, prevCR); err != nil { return err } - if err := createOrUpdateVMSelectHPA(ctx, rclient, cr, prevCR); err != nil { return err } if err := createOrUpdateVMSelectVPA(ctx, rclient, cr, prevCR); err != nil { return err } - // create vmselect service if err := createOrUpdateVMSelectService(ctx, rclient, cr, prevCR); err != nil { return err } } - - if cr.Spec.VMInsert != nil { - if cr.Spec.VMInsert.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if cr.Spec.VMInsert.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if err := createOrUpdateVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertService(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertHPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertVPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if prevCR != nil { if err := deleteOrphaned(ctx, rclient, cr); err != nil { return fmt.Errorf("failed to remove objects from previous cluster state: %w", err) @@ -321,21 +271,22 @@ func createOrUpdateLBProxyService(ctx context.Context, rclient client.Client, cr return nil } -func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string, owner metav1.OwnerReference) error { var prevDeploy *appsv1.Deployment if prevCR != nil && prevCR.Spec.VMInsert != nil { var err error - prevDeploy, err = genVMInsertSpec(prevCR) + prevDeploy, err = genVMInsertSpec(prevCR, poolName) if err != nil { return fmt.Errorf("cannot generate prev deploy spec: %w", err) } + prevDeploy.OwnerReferences = []metav1.OwnerReference{owner} } - newDeployment, err := genVMInsertSpec(cr) + newDeployment, err := genVMInsertSpec(cr, poolName) if err != nil { return err } - owner := cr.AsOwner() + newDeployment.OwnerReferences = []metav1.OwnerReference{owner} o := reconcile.DeploymentOpts{ PatchSpec: func(existingSpec, newSpec *appsv1.DeploymentSpec) { if cr.Spec.VMInsert.HPA != nil { @@ -347,8 +298,8 @@ func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prev return reconcile.Deployment(ctx, rclient, newDeployment, prevDeploy, &owner, &o) } -func buildVMInsertService(cr *vmv1beta1.VMCluster) *corev1.Service { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func buildVMInsertService(cr *vmv1beta1.VMCluster, poolName string) *corev1.Service { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) svc := build.Service(b, cr.Spec.VMInsert.Port, func(svc *corev1.Service) { build.AppendInsertPortsToService(cr.Spec.VMInsert.InsertPorts, svc) if cr.Spec.VMInsert.ClusterNativePort != "" { @@ -381,16 +332,17 @@ func buildVMInsertScrape(cr *vmv1beta1.VMCluster, svc *corev1.Service) *vmv1beta return svs } -func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - svc := buildVMInsertService(cr) +func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, owner metav1.OwnerReference, poolName string) error { + svc := buildVMInsertService(cr, poolName) + svc.OwnerReferences = []metav1.OwnerReference{owner} var prevSvc, prevAdditionalSvc *corev1.Service if prevCR != nil && prevCR.Spec.VMInsert != nil { - prevSvc = buildVMInsertService(prevCR) + prevSvc = buildVMInsertService(prevCR, poolName) + prevSvc.OwnerReferences = []metav1.OwnerReference{owner} prevAdditionalSvcBase := *prevSvc prevAdditionalSvcBase.Name = prevCR.PrefixedName(vmv1beta1.ClusterComponentInsert) prevAdditionalSvc = build.AdditionalServiceFromDefault(&prevAdditionalSvcBase, prevCR.Spec.VMInsert.ServiceSpec) } - owner := cr.AsOwner() if err := cr.Spec.VMInsert.ServiceSpec.IsSomeAndThen(func(s *vmv1beta1.AdditionalServiceSpec) error { additionalSvcBase := *svc additionalSvcBase.Name = cr.PrefixedName(vmv1beta1.ClusterComponentInsert) @@ -431,23 +383,25 @@ func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, c return nil } -func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string, owner metav1.OwnerReference) error { var prevSts *appsv1.StatefulSet if prevCR != nil && prevCR.Spec.VMStorage != nil { var err error - prevSts, err = buildVMStorageSpec(ctx, prevCR) + prevSts, err = buildVMStorageSpec(ctx, prevCR, poolName) if err != nil { return fmt.Errorf("cannot build prev storage spec: %w", err) } + prevSts.OwnerReferences = []metav1.OwnerReference{owner} } - newSts, err := buildVMStorageSpec(ctx, cr) + newSts, err := buildVMStorageSpec(ctx, cr, poolName) if err != nil { return err } + newSts.OwnerReferences = []metav1.OwnerReference{owner} o := reconcile.StatefulSetOpts{ - SelectorLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentStorage), + SelectorLabels: newSts.Spec.Selector.MatchLabels, UpdateBehavior: cr.Spec.VMStorage.RollingUpdateStrategyBehavior, PatchSpec: func(existingSpec, newSpec *appsv1.StatefulSetSpec) { if cr.Spec.VMStorage.HPA != nil { @@ -456,12 +410,11 @@ func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, pre } }, } - owner := cr.AsOwner() return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o) } -func buildVMStorageService(cr *vmv1beta1.VMCluster) *corev1.Service { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func buildVMStorageService(cr *vmv1beta1.VMCluster, poolName string) *corev1.Service { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) return build.Service(b, cr.Spec.VMStorage.Port, func(svc *corev1.Service) { svc.Spec.ClusterIP = "None" svc.Spec.PublishNotReadyAddresses = true @@ -498,14 +451,15 @@ func buildVMStorageScrape(cr *vmv1beta1.VMCluster, svc *corev1.Service) *vmv1bet return build.VMServiceScrape(svc, cr.Spec.VMStorage, "vmbackupmanager") } -func createOrUpdateVMStorageService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - svc := buildVMStorageService(cr) +func createOrUpdateVMStorageService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, owner metav1.OwnerReference, poolName string) error { + svc := buildVMStorageService(cr, poolName) + svc.OwnerReferences = []metav1.OwnerReference{owner} var prevSvc, prevAdditionalSvc *corev1.Service if prevCR != nil && prevCR.Spec.VMStorage != nil { - prevSvc = buildVMStorageService(prevCR) + prevSvc = buildVMStorageService(prevCR, poolName) + prevSvc.OwnerReferences = []metav1.OwnerReference{owner} prevAdditionalSvc = build.AdditionalServiceFromDefault(prevSvc, prevCR.Spec.VMStorage.ServiceSpec) } - owner := cr.AsOwner() if err := cr.Spec.VMStorage.ServiceSpec.IsSomeAndThen(func(s *vmv1beta1.AdditionalServiceSpec) error { additionalSvc := build.AdditionalServiceFromDefault(svc, s) if additionalSvc.Name == svc.Name { @@ -623,30 +577,59 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } storageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if d := cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { - storageNode := fmt.Sprintf("srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)) - for _, node := range cr.Spec.VMSelect.ExtraStorageNodes { - storageNode += "," + node.Addr + storageNodeFlag := build.NewFlag("-storageNode", "") + var baseCount int + d := cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery) + if len(cr.Spec.Pools) == 0 { + if d != nil && d.Enabled { + storageNodeFlag.Add(fmt.Sprintf("srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)), 0) + baseCount = 1 + } else { + storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentSelect) + for idx, i := range storageNodeIds { + storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName), idx) + } + baseCount = len(storageNodeIds) } - args = append(args, fmt.Sprintf("-storageNode=%s", storageNode)) + } + + // Pool storage nodes — each pool is exposed as a named storage group. + poolCount := baseCount + for _, pool := range cr.Spec.Pools { + poolStr, err := poolStorage(cr, &pool) + if err != nil { + return nil, fmt.Errorf("pool %q: cannot build storage spec: %w", pool.Name, err) + } + if poolStr == nil { + continue + } + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + if d != nil && d.Enabled { + storageNodeFlag.Add(fmt.Sprintf("%s/srv+%s", pool.Name, storageNodeSRVAddr(poolStorageName, cr.Namespace, poolStr.VMSelectPort, cr.Spec.ClusterDomainName)), poolCount) + poolCount++ + } else { + for _, i := range storageNodeIDs(poolStr, vmv1beta1.ClusterComponentSelect) { + addr := vmv1beta1.PodDNSAddress(poolStorageName, i, cr.Namespace, poolStr.VMSelectPort, cr.Spec.ClusterDomainName) + storageNodeFlag.Add(fmt.Sprintf("%s/%s", pool.Name, addr), poolCount) + poolCount++ + } + } + } + + if d != nil && d.Enabled { if d.Interval != "" { args = append(args, fmt.Sprintf("-storageNode.discoveryInterval=%s", d.Interval)) } if d.Filter != "" { args = append(args, fmt.Sprintf("-storageNode.filter=%s", d.Filter)) } - } else { - storageNodeFlag := build.NewFlag("-storageNode", "") - storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentSelect) - for idx, i := range storageNodeIds { - storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName), idx) - } - for i, node := range cr.Spec.VMSelect.ExtraStorageNodes { - storageNodeFlag.Add(node.Addr, i+len(storageNodeIds)) - } - args = build.AppendFlagsToArgs(args, len(storageNodeIds)+len(cr.Spec.VMSelect.ExtraStorageNodes), storageNodeFlag) } + for i, node := range cr.Spec.VMSelect.ExtraStorageNodes { + storageNodeFlag.Add(node.Addr, poolCount+i) + } + args = build.AppendFlagsToArgs(args, poolCount+len(cr.Spec.VMSelect.ExtraStorageNodes), storageNodeFlag) + // selectNode arg add for deployments without HPA // HPA leads to rolling restart for vmselect statefulset in case of replicas count changes if cr.Spec.VMSelect.HPA == nil && cr.Spec.VMSelect.ReplicaCount != nil { @@ -775,6 +758,9 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } func createOrUpdatePodDisruptionBudgetForVMSelect(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.VMSelect.PodDisruptionBudget == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentSelect) pdb := build.PodDisruptionBudget(b, cr.Spec.VMSelect.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget @@ -787,6 +773,9 @@ func createOrUpdatePodDisruptionBudgetForVMSelect(ctx context.Context, rclient c } func createOrUpdateNetworkPolicyForVMSelect(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.VMSelect.NetworkPolicy == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentSelect) np := build.NetworkPolicy(b, cr.Spec.VMSelect.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy @@ -798,23 +787,26 @@ func createOrUpdateNetworkPolicyForVMSelect(ctx context.Context, rclient client. return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { - - podSpec, err := makePodSpecForVMInsert(cr) +func genVMInsertSpec(cr *vmv1beta1.VMCluster, poolName string) (*appsv1.Deployment, error) { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) + podSpec, err := makePodSpecForVMInsert(cr, poolName) if err != nil { return nil, err } + for k, v := range b.SelectorLabels() { + podSpec.Labels[k] = v + } strategyType := appsv1.RollingUpdateDeploymentStrategyType if cr.Spec.VMInsert.UpdateStrategy != nil { strategyType = *cr.Spec.VMInsert.UpdateStrategy } - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentInsert) + commonName := b.PrefixedName() stsSpec := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: commonName, Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Labels: b.FinalLabels(), Annotations: cr.FinalAnnotations(), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, @@ -827,7 +819,7 @@ func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { RollingUpdate: cr.Spec.VMInsert.RollingUpdate, }, Selector: &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentInsert), + MatchLabels: b.SelectorLabels(), }, Template: *podSpec, }, @@ -836,7 +828,7 @@ func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { return stsSpec, nil } -func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, error) { +func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster, poolName string) (*corev1.PodTemplateSpec, error) { cfg := config.MustGetBaseConfig() args := []string{ fmt.Sprintf("-httpListenAddr=:%s", cr.Spec.VMInsert.Port), @@ -857,21 +849,49 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } storageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if d := cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { - args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName))) - if d.Interval != "" { - args = append(args, fmt.Sprintf("-storageNode.discoveryInterval=%s", d.Interval)) + if poolName != "" { + storageName = cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName) + } + if len(cr.Spec.Pools) == 0 { + if d := cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { + args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName))) + if d.Interval != "" { + args = append(args, fmt.Sprintf("-storageNode.discoveryInterval=%s", d.Interval)) + } + if d.Filter != "" { + args = append(args, fmt.Sprintf("-storageNode.filter=%s", d.Filter)) + } + } else { + storageNodeFlag := build.NewFlag("-storageNode", "") + storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentInsert) + for idx, i := range storageNodeIds { + storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName), idx) + } + args = build.AppendFlagsToArgs(args, len(storageNodeIds), storageNodeFlag) } - if d.Filter != "" { - args = append(args, fmt.Sprintf("-storageNode.filter=%s", d.Filter)) + } + + // Pool storage nodes for pools using the shared vminsert (no dedicated insert). + for _, pool := range cr.Spec.Pools { + if pool.VMInsert != nil { + continue } - } else { - storageNodeFlag := build.NewFlag("-storageNode", "") - storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentInsert) - for idx, i := range storageNodeIds { - storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName), idx) + poolStr, err := poolStorage(cr, &pool) + if err != nil { + return nil, fmt.Errorf("pool %q: cannot build storage spec: %w", pool.Name, err) + } + if poolStr == nil { + continue + } + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + if d := cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { + args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(poolStorageName, cr.Namespace, poolStr.VMInsertPort, cr.Spec.ClusterDomainName))) + } else { + for _, i := range storageNodeIDs(poolStr, vmv1beta1.ClusterComponentInsert) { + addr := vmv1beta1.PodDNSAddress(poolStorageName, i, cr.Namespace, poolStr.VMInsertPort, cr.Spec.ClusterDomainName) + args = append(args, fmt.Sprintf("-storageNode=%s", addr)) + } } - args = build.AppendFlagsToArgs(args, len(storageNodeIds), storageNodeFlag) } if cr.Spec.ReplicationFactor != nil { @@ -997,49 +1017,60 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e return vmInsertPodSpec, nil } -func createOrUpdatePodDisruptionBudgetForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func createOrUpdatePodDisruptionBudgetForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMInsert.PodDisruptionBudget == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) pdb := build.PodDisruptionBudget(b, cr.Spec.VMInsert.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget if prevCR != nil && prevCR.Spec.VMInsert.PodDisruptionBudget != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevPDB = build.PodDisruptionBudget(b, prevCR.Spec.VMInsert.PodDisruptionBudget) } owner := cr.AsOwner() return reconcile.PDB(ctx, rclient, pdb, prevPDB, &owner) } -func createOrUpdateNetworkPolicyForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func createOrUpdateNetworkPolicyForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMInsert.NetworkPolicy == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) np := build.NetworkPolicy(b, cr.Spec.VMInsert.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.NetworkPolicy != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevNP = build.NetworkPolicy(b, prevCR.Spec.VMInsert.NetworkPolicy) } owner := cr.AsOwner() return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func buildVMStorageSpec(ctx context.Context, cr *vmv1beta1.VMCluster) (*appsv1.StatefulSet, error) { - - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) +func buildVMStorageSpec(ctx context.Context, cr *vmv1beta1.VMCluster, poolName string) (*appsv1.StatefulSet, error) { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) + commonName := b.PrefixedName() podSpec, err := makePodSpecForVMStorage(ctx, cr) if err != nil { return nil, err } + // Merge selector labels into pod template so the STS selector matches its pods. + // For pools this adds the pool label required to keep per-pool selectors disjoint. + for k, v := range b.SelectorLabels() { + podSpec.Labels[k] = v + } stsSpec := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: commonName, Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentStorage), + Labels: b.FinalLabels(), Annotations: cr.FinalAnnotations(), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentStorage), + MatchLabels: b.SelectorLabels(), }, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: cr.Spec.VMStorage.RollingUpdateStrategy, @@ -1071,7 +1102,9 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor if cfg.EnableTCP6 { args = append(args, "-enableTCP6") } - if cr.Spec.RetentionPeriod != "" { + if rp := cr.Spec.VMStorage.RetentionPeriod; rp != "" { + args = append(args, fmt.Sprintf("-retentionPeriod=%s", rp)) + } else if cr.Spec.RetentionPeriod != "" { args = append(args, fmt.Sprintf("-retentionPeriod=%s", cr.Spec.RetentionPeriod)) } @@ -1279,35 +1312,41 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor return vmStoragePodSpec, nil } -func createOrUpdatePodDisruptionBudgetForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func createOrUpdatePodDisruptionBudgetForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMStorage.PodDisruptionBudget == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) pdb := build.PodDisruptionBudget(b, cr.Spec.VMStorage.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget if prevCR != nil && prevCR.Spec.VMStorage.PodDisruptionBudget != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevPDB = build.PodDisruptionBudget(b, prevCR.Spec.VMStorage.PodDisruptionBudget) } owner := cr.AsOwner() return reconcile.PDB(ctx, rclient, pdb, prevPDB, &owner) } -func createOrUpdateNetworkPolicyForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func createOrUpdateNetworkPolicyForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMStorage.NetworkPolicy == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) np := build.NetworkPolicy(b, cr.Spec.VMStorage.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.NetworkPolicy != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevNP = build.NetworkPolicy(b, prevCR.Spec.VMStorage.NetworkPolicy) } owner := cr.AsOwner() return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { if cr.Spec.VMInsert.HPA == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) targetRef := autoscalingv2.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "Deployment", @@ -1316,7 +1355,7 @@ func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, p newHPA := build.HPA(b, targetRef, cr.Spec.VMInsert.HPA) var prevHPA *autoscalingv2.HorizontalPodAutoscaler if prevCR != nil && prevCR.Spec.VMInsert.HPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevHPA = build.HPA(b, targetRef, prevCR.Spec.VMInsert.HPA) } owner := cr.AsOwner() @@ -1343,12 +1382,12 @@ func createOrUpdateVMSelectHPA(ctx context.Context, rclient client.Client, cr, p return reconcile.HPA(ctx, rclient, defaultHPA, prevHPA, &owner) } -func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { hpa := cr.Spec.VMStorage.HPA if hpa == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) targetRef := autoscalingv2.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "StatefulSet", @@ -1357,18 +1396,18 @@ func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, defaultHPA := build.HPA(b, targetRef, hpa) var prevHPA *autoscalingv2.HorizontalPodAutoscaler if prevCR != nil && prevCR.Spec.VMStorage.HPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevHPA = build.HPA(b, targetRef, prevCR.Spec.VMStorage.HPA) } owner := cr.AsOwner() return reconcile.HPA(ctx, rclient, defaultHPA, prevHPA, &owner) } -func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { if cr.Spec.VMInsert.VPA == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) targetRef := autoscalingv1.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "Deployment", @@ -1377,7 +1416,7 @@ func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, p newVPA := build.VPA(b, targetRef, cr.Spec.VMInsert.VPA) var prevVPA *vpav1.VerticalPodAutoscaler if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.VPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevVPA = build.VPA(b, targetRef, prevCR.Spec.VMInsert.VPA) } owner := cr.AsOwner() @@ -1404,12 +1443,12 @@ func createOrUpdateVMSelectVPA(ctx context.Context, rclient client.Client, cr, p return reconcile.VPA(ctx, rclient, newVPA, prevVPA, &owner) } -func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { vpa := cr.Spec.VMStorage.VPA if vpa == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) targetRef := autoscalingv1.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "StatefulSet", @@ -1418,7 +1457,7 @@ func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, newVPA := build.VPA(b, targetRef, vpa) var prevVPA *vpav1.VerticalPodAutoscaler if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.VPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevVPA = build.VPA(b, targetRef, prevCR.Spec.VMStorage.VPA) } owner := cr.AsOwner() @@ -1431,34 +1470,62 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM newInsert := cr.Spec.VMInsert newLB := cr.Spec.RequestsLoadBalancer + hasPools := len(cr.Spec.Pools) > 0 + hasPoolInsert := false + for _, p := range cr.Spec.Pools { + if p.VMInsert != nil { + hasPoolInsert = true + break + } + } + cc := finalize.NewChildCleaner() - if newStorage == nil { - if err := finalize.OnStorageDelete(ctx, rclient, cr, true); err != nil { - return fmt.Errorf("cannot remove orphaned storage resources: %w", err) + + keepStorageResources := func(name string, storage *vmv1beta1.VMStorage) { + cc.KeepService(name) + if storage.ServiceSpec != nil && !storage.ServiceSpec.UseAsDefault { + cc.KeepService(storage.ServiceSpec.NameOrDefault(name)) } - } else { - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if newStorage.PodDisruptionBudget != nil { - cc.KeepPDB(commonName) + if !ptr.Deref(storage.DisableSelfServiceScrape, false) { + cc.KeepScrape(name) } - if newStorage.NetworkPolicy != nil { - cc.KeepNetworkPolicy(commonName) + if storage.PodDisruptionBudget != nil { + cc.KeepPDB(name) } - if newStorage.HPA != nil { - cc.KeepHPA(commonName) + if storage.HPA != nil { + cc.KeepHPA(name) } - if newStorage.VPA != nil { - cc.KeepVPA(commonName) + if storage.VPA != nil { + cc.KeepVPA(name) } - if !ptr.Deref(newStorage.DisableSelfServiceScrape, false) { - cc.KeepScrape(commonName) + } + keepInsertResources := func(name, scrapeName string, insert *vmv1beta1.VMInsert) { + cc.KeepService(name) + if insert.ServiceSpec != nil && !insert.ServiceSpec.UseAsDefault { + cc.KeepService(insert.ServiceSpec.NameOrDefault(name)) } - cc.KeepService(commonName) - if newStorage.ServiceSpec != nil && !newStorage.ServiceSpec.UseAsDefault { - cc.KeepService(newStorage.ServiceSpec.NameOrDefault(commonName)) + if !ptr.Deref(insert.DisableSelfServiceScrape, false) { + cc.KeepScrape(scrapeName) + } + if insert.PodDisruptionBudget != nil { + cc.KeepPDB(name) + } + if insert.HPA != nil { + cc.KeepHPA(name) + } + if insert.VPA != nil { + cc.KeepVPA(name) } } + if newStorage == nil && !hasPools { + if err := finalize.OnStorageDelete(ctx, rclient, cr, true); err != nil { + return fmt.Errorf("cannot remove orphaned storage resources: %w", err) + } + } else if newStorage != nil { + keepStorageResources(cr.PrefixedName(vmv1beta1.ClusterComponentStorage), newStorage) + } + if newSelect == nil { if err := finalize.OnSelectDelete(ctx, rclient, cr, true); err != nil { return fmt.Errorf("cannot remove orphaned select resources: %w", err) @@ -1491,36 +1558,18 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM } } - if newInsert == nil { + if newInsert == nil && !hasPoolInsert { if err := finalize.OnInsertDelete(ctx, rclient, cr, true); err != nil { return fmt.Errorf("cannot remove orphaned insert resources: %w", err) } - } else { + } else if newInsert != nil { commonName := cr.PrefixedName(vmv1beta1.ClusterComponentInsert) - if newInsert.PodDisruptionBudget != nil { - cc.KeepPDB(commonName) - } - if newInsert.NetworkPolicy != nil { - cc.KeepNetworkPolicy(commonName) - } - if newInsert.HPA != nil { - cc.KeepHPA(commonName) - } - if newInsert.VPA != nil { - cc.KeepVPA(commonName) - } - cc.KeepService(commonName) - if newInsert.ServiceSpec != nil && !newInsert.ServiceSpec.UseAsDefault { - cc.KeepService(newInsert.ServiceSpec.NameOrDefault(commonName)) - } scrapeName := commonName if newLB.Enabled && !newLB.DisableInsertBalancing { scrapeName = cr.PrefixedInternalName(vmv1beta1.ClusterComponentInsert) cc.KeepService(scrapeName) } - if !ptr.Deref(newInsert.DisableSelfServiceScrape, false) { - cc.KeepScrape(scrapeName) - } + keepInsertResources(commonName, scrapeName, newInsert) } if newLB.Enabled { commonName := cr.PrefixedName(vmv1beta1.ClusterComponentBalancer) @@ -1553,6 +1602,47 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM return fmt.Errorf("cannot remove serviceaccount: %w", err) } } + + // Pool resource cleanup: keep active pool services and remove orphaned pool StatefulSets/Deployments. + keepStorageSTSNames := sets.New[string]() + keepInsertDeploymentNames := sets.New[string]() + if newStorage != nil { + keepStorageSTSNames.Insert(cr.PrefixedName(vmv1beta1.ClusterComponentStorage)) + } + if newInsert != nil { + keepInsertDeploymentNames.Insert(cr.PrefixedName(vmv1beta1.ClusterComponentInsert)) + } + for _, pool := range cr.Spec.Pools { + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + keepStorageSTSNames.Insert(poolStorageName) + poolStorageItem, err := poolStorage(cr, &pool) + if err != nil { + return fmt.Errorf("pool %q: cannot resolve storage: %w", pool.Name, err) + } + if poolStorageItem != nil { + keepStorageResources(poolStorageName, poolStorageItem) + } + if pool.VMInsert != nil { + poolInsertName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, pool.Name) + keepInsertDeploymentNames.Insert(poolInsertName) + poolInsertItem, err := poolInsert(cr, pool) + if err != nil { + return fmt.Errorf("pool %q: cannot resolve insert: %w", pool.Name, err) + } + if poolInsertItem != nil { + keepInsertResources(poolInsertName, poolInsertName, poolInsertItem) + } + } + } + storageBuilder := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + if err := finalize.RemoveOrphanedSTSs(ctx, rclient, storageBuilder, keepStorageSTSNames, true); err != nil { + return fmt.Errorf("cannot remove orphaned pool storage StatefulSets: %w", err) + } + insertBuilder := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + if err := finalize.RemoveOrphanedDeployments(ctx, rclient, insertBuilder, keepInsertDeploymentNames, true); err != nil { + return fmt.Errorf("cannot remove orphaned pool insert Deployments: %w", err) + } + return cc.RemoveOrphaned(ctx, rclient, cr) } @@ -1788,15 +1878,11 @@ func createOrUpdateVMAuthLB(ctx context.Context, rclient client.Client, cr, prev if err := createOrUpdateVMAuthLBService(ctx, rclient, cr, prevCR); err != nil { return err } - if cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { - return fmt.Errorf("cannot create or update PodDisruptionBudget for vmauth lb: %w", err) - } + if err := createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { + return fmt.Errorf("cannot create or update PodDisruptionBudget for vmauth lb: %w", err) } - if cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { - return fmt.Errorf("cannot create or update NetworkPolicy for vmauth lb: %w", err) - } + if err := createOrUpdateNetworkPolicyForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { + return fmt.Errorf("cannot create or update NetworkPolicy for vmauth lb: %w", err) } if err := createOrUpdateVMAuthLBHPA(ctx, rclient, cr, prevCR); err != nil { return fmt.Errorf("cannot create or update HPA for vmauth lb: %w", err) @@ -1834,6 +1920,9 @@ func storageNodeSRVAddr(svcName, namespace, port, clusterDomain string) string { } func createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentBalancer) pdb := build.PodDisruptionBudget(b, cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget @@ -1846,6 +1935,9 @@ func createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx context.Context, rclient c } func createOrUpdateNetworkPolicyForVMAuthLB(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentBalancer) np := build.NetworkPolicy(b, cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_pools.go b/internal/controller/operator/factory/vmcluster/vmcluster_pools.go new file mode 100644 index 0000000000..1a08e00084 --- /dev/null +++ b/internal/controller/operator/factory/vmcluster/vmcluster_pools.go @@ -0,0 +1,205 @@ +package vmcluster + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" +) + +// createOrUpdatePool reconciles all resources for a single pool in sequence: +// vmstorage StatefulSet → vmstorage Service → vminsert Deployment + Service (if dedicated). +// pool is nil for the base (non-pool) cluster, in which case cr's own top-level +// VMStorage/VMInsert are treated as that "pool"'s definition. +func createOrUpdatePool(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool, owner metav1.OwnerReference) error { + var poolName string + view, err := buildPoolView(cr, pool) + if err != nil { + return err + } + poolVMStorage := cr.Spec.VMStorage + poolVMInsert := cr.Spec.VMInsert + if pool != nil { + poolName = pool.Name + // every declared pool gets its own storage, falling back to the base + // config; insert stays shared unless the pool overrides it. + poolVMStorage = view.Spec.VMStorage + poolVMInsert = pool.VMInsert + } else { + if len(cr.Spec.Pools) > 0 { + poolVMStorage = nil + } + for _, p := range cr.Spec.Pools { + if p.VMInsert != nil { + poolVMInsert = nil + break + } + } + } + var prevView *vmv1beta1.VMCluster + if prevCR != nil { + if pool == nil { + prevView, err = buildPoolView(prevCR, nil) + if err != nil { + return err + } + } else if prevPool, ok := findPool(prevCR.Spec.Pools, poolName); ok { + prevView, err = buildPoolView(prevCR, &prevPool) + if err != nil { + return err + } + } + } + if poolVMStorage != nil { + if err = createOrUpdatePodDisruptionBudgetForVMStorage(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage pdb: %w", err) + } + if err = createOrUpdateVMStorage(ctx, rclient, view, prevView, poolName, owner); err != nil { + return fmt.Errorf("vmstorage: %w", err) + } + if err = createOrUpdateVMStorageService(ctx, rclient, view, prevView, owner, poolName); err != nil { + return fmt.Errorf("vmstorage service: %w", err) + } + if err = createOrUpdateVMStorageHPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage hpa: %w", err) + } + if err = createOrUpdateVMStorageVPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage vpa: %w", err) + } + if err = createOrUpdateNetworkPolicyForVMStorage(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage networkPolicy: %w", err) + } + } + if poolVMInsert != nil { + if err = createOrUpdatePodDisruptionBudgetForVMInsert(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert pdb: %w", err) + } + if err = createOrUpdateVMInsert(ctx, rclient, view, prevView, poolName, owner); err != nil { + return fmt.Errorf("vminsert: %w", err) + } + if err = createOrUpdateVMInsertService(ctx, rclient, view, prevView, owner, poolName); err != nil { + return fmt.Errorf("vminsert service: %w", err) + } + if err = createOrUpdateVMInsertHPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert hpa: %w", err) + } + if err = createOrUpdateVMInsertVPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert vpa: %w", err) + } + if err = createOrUpdateNetworkPolicyForVMInsert(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert networkPolicy: %w", err) + } + } + return nil +} + +// buildPoolView returns a deep copy of cr with VMStorage/VMInsert/RetentionPeriod substituted +// with the pool's merged values (pool == nil leaves the base cluster's own values as-is). +// view.Name stays equal to cr.Name so label methods still produce the correct instance label; +// pool-specific naming/labels are applied separately via build.NewPoolBuilder. +func buildPoolView(cr *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool) (*vmv1beta1.VMCluster, error) { + view := cr.DeepCopy() + storage, err := poolStorage(cr, pool) + if err != nil { + return nil, fmt.Errorf("cannot resolve vmstorage: %w", err) + } + view.Spec.VMStorage = storage + if storage != nil && storage.RetentionPeriod != "" { + view.Spec.RetentionPeriod = storage.RetentionPeriod + } + + if pool != nil { + if pool.VMInsert != nil { + insert, err := poolInsert(cr, *pool) + if err != nil { + return nil, fmt.Errorf("cannot resolve vminsert: %w", err) + } + view.Spec.VMInsert = insert + } else { + view.Spec.VMInsert = nil + } + // a per-pool view must not see sibling pools. + view.Spec.Pools = nil + } + + view.Spec.VMSelect = nil + return view, nil +} + +// poolStorage merges the pool's vmstorage over the top-level base (pool fields win, absent +// fields fall through). pool == nil, or a pool with no VMStorage override, both resolve to the +// base as-is. +func poolStorage(cr *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool) (*vmv1beta1.VMStorage, error) { + base := cr.Spec.VMStorage + if pool == nil || pool.VMStorage == nil { + if base == nil { + return nil, nil + } + return base.DeepCopy(), nil + } + merged := pool.VMStorage.DeepCopy() + if base != nil { + // reverse=true: merged (pool) fields win; base fills absent fields. + if err := vmv1beta1.MergeDeep(merged, base, true); err != nil { + return nil, err + } + } + return merged, nil +} + +// poolInsert merges the pool's vminsert over the top-level base using MergeDeep. +func poolInsert(cr *vmv1beta1.VMCluster, pool vmv1beta1.VMClusterPool) (*vmv1beta1.VMInsert, error) { + if pool.VMInsert == nil { + return nil, nil + } + base := cr.Spec.VMInsert + merged := pool.VMInsert.DeepCopy() + if base != nil { + if err := vmv1beta1.MergeDeep(merged, base, true); err != nil { + return nil, err + } + } + return merged, nil +} + +// storageNodeIDs returns available storage node IDs for the given VMStorage spec, +// skipping nodes under maintenance for the given component kind. +func storageNodeIDs(storage *vmv1beta1.VMStorage, kind vmv1beta1.ClusterComponent) []int32 { + if storage == nil || (storage.ReplicaCount == nil && storage.HPA == nil) { + return nil + } + maintenanceNodes := sets.New[int32]() + switch kind { + case vmv1beta1.ClusterComponentSelect: + maintenanceNodes.Insert(storage.MaintenanceSelectNodeIDs...) + case vmv1beta1.ClusterComponentInsert: + maintenanceNodes.Insert(storage.MaintenanceInsertNodeIDs...) + } + var replicaCount int32 + if storage.ReplicaCount != nil { + replicaCount = *storage.ReplicaCount + } else if storage.HPA != nil { + replicaCount = storage.HPA.GetMinReplicas() + } + var result []int32 + for i := int32(0); i < replicaCount; i++ { + if !maintenanceNodes.Has(i) { + result = append(result, i) + } + } + return result +} + +// findPool returns the pool with the given name and whether it was found. +func findPool(pools []vmv1beta1.VMClusterPool, name string) (vmv1beta1.VMClusterPool, bool) { + for _, p := range pools { + if p.Name == name { + return p, true + } + } + return vmv1beta1.VMClusterPool{}, false +} diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go index a730a772bb..b439ccb3d6 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go @@ -167,6 +167,16 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, {Verb: "Create", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert - pools reconcile vmstorage and vminsert together per pool, so + // vminsert now runs right after vmstorage, ahead of vmselect. + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Create", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, // wait for ready + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Create", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + {Verb: "Create", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Create", Kind: "StatefulSet", Resource: vmselectName}, @@ -175,15 +185,6 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Create", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, {Verb: "Create", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Create", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, // wait for ready - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Create", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, - {Verb: "Create", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) @@ -203,18 +204,18 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "Service", Resource: vmstorageName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // getLatestStsState {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // patchSTSCurrentRevision {Verb: "Get", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) @@ -239,18 +240,18 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "Service", Resource: vmstorageName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // getLatestStsState {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // patchSTSCurrentRevision {Verb: "Get", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) } diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index 33f3d57dea..e659db762f 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -13,6 +13,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,6 +30,16 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) +// containerArgs returns the Args of the named container in spec, or nil if absent. +func containerArgs(spec corev1.PodSpec, name string) []string { + for _, c := range spec.Containers { + if c.Name == name { + return c.Args + } + } + return nil +} + func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMCluster @@ -858,6 +869,198 @@ func TestCreateOrUpdate(t *testing.T) { "managed-by": "vm-operator", }, svc.Labels) }}) + + // pools: two pools with shared vminsert — pool STSes get pool label, instance label stays the cluster name, top-level vmstorage not created + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot"}, + {Name: "cold"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + for _, poolName := range []string{"hot", "cold"} { + stsName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName) + var sts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: stsName}, &sts), "pool STS %s should exist", stsName) + // instance label must be the cluster name, not cluster-pool + assert.Equal(t, cr.Name, sts.Labels["app.kubernetes.io/instance"], "instance label for pool %s", poolName) + // STS selector must include the pool label so per-pool selectors are disjoint + assert.Equal(t, poolName, sts.Spec.Selector.MatchLabels["app.kubernetes.io/pool"], "selector pool label for pool %s", poolName) + // pod template must also carry the pool label so the STS selector matches its pods + assert.Equal(t, poolName, sts.Spec.Template.Labels["app.kubernetes.io/pool"], "pod template pool label for pool %s", poolName) + } + // top-level vmstorage must NOT be created when pools are defined + var topSts appsv1.StatefulSet + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, &topSts) + assert.True(t, k8serrors.IsNotFound(err), "top-level vmstorage STS must not exist when pools are defined") + // shared vminsert must still be created (no pool has a dedicated insert) + var dep appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, &dep)) + // shared vminsert must route to both pools' own storage nodes + args := containerArgs(dep.Spec.Template.Spec, "vminsert") + hotStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + coldStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "cold") + var hasHot, hasCold bool + for _, a := range args { + if !strings.HasPrefix(a, "-storageNode=") { + continue + } + hasHot = hasHot || strings.Contains(a, hotStorageName) + hasCold = hasCold || strings.Contains(a, coldStorageName) + } + assert.True(t, hasHot, "vminsert args should reference hot pool storage, got %v", args) + assert.True(t, hasCold, "vminsert args should reference cold pool storage, got %v", args) + }, + }) + + // pools: pool with dedicated vminsert — pool insert created, top-level vminsert skipped + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + // pool insert must exist with pool-scoped name + poolInsertName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, "hot") + var poolDep appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: poolInsertName}, &poolDep)) + assert.Equal(t, cr.Name, poolDep.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", poolDep.Labels["app.kubernetes.io/pool"]) + // a pool's dedicated vminsert must route to its own storage nodes, not the + // (nonexistent) top-level storage StatefulSet + args := containerArgs(poolDep.Spec.Template.Spec, "vminsert") + hotStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + baseStorageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) + var hasHot bool + for _, a := range args { + if !strings.HasPrefix(a, "-storageNode=") { + continue + } + assert.False(t, strings.Contains(a, baseStorageName) && !strings.Contains(a, hotStorageName), + "pool vminsert must not reference the top-level storage name, got %q", a) + hasHot = hasHot || strings.Contains(a, hotStorageName) + } + assert.True(t, hasHot, "pool vminsert args should reference hot pool storage, got %v", args) + // top-level vminsert must NOT be created when any pool has its own insert + var topDep appsv1.Deployment + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, &topDep) + assert.True(t, k8serrors.IsNotFound(err), "top-level vminsert must not exist when a pool has a dedicated insert") + }, + }) + + // pools: NetworkPolicy for a pool's vmstorage/vminsert must be pool-scoped (name, selector, + // and labels), not collide with the base cluster's NetworkPolicy naming + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + RetentionPeriod: "1", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + NetworkPolicy: &vmv1beta1.EmbeddedNetworkPolicy{}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + NetworkPolicy: &vmv1beta1.EmbeddedNetworkPolicy{}, + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + storageNPName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + var storageNP networkingv1.NetworkPolicy + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: storageNPName}, &storageNP)) + assert.Equal(t, cr.Name, storageNP.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", storageNP.Labels["app.kubernetes.io/pool"]) + assert.Equal(t, "hot", storageNP.Spec.PodSelector.MatchLabels["app.kubernetes.io/pool"]) + + insertNPName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, "hot") + var insertNP networkingv1.NetworkPolicy + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: insertNPName}, &insertNP)) + assert.Equal(t, cr.Name, insertNP.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", insertNP.Labels["app.kubernetes.io/pool"]) + assert.Equal(t, "hot", insertNP.Spec.PodSelector.MatchLabels["app.kubernetes.io/pool"]) + }, + }) + + // pools: VMStorage.RetentionPeriod overrides cluster-level RetentionPeriod in the generated args + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + RetentionPeriod: "90d", + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var sts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, &sts)) + var storageArgs []string + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "vmstorage" { + storageArgs = c.Args + } + } + hasRetention90d := false + hasRetention1 := false + for _, a := range storageArgs { + if a == "-retentionPeriod=90d" { + hasRetention90d = true + } + if a == "-retentionPeriod=1" { + hasRetention1 = true + } + } + assert.True(t, hasRetention90d, "VMStorage.RetentionPeriod should override cluster-level: got args %v", storageArgs) + assert.False(t, hasRetention1, "cluster-level RetentionPeriod should be overridden: got args %v", storageArgs) + }, + }) } func TestCreatOrUpdateClusterServices(t *testing.T) { @@ -1961,6 +2164,91 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { }, ) + // pools: vmselect gets pool-grouped storage nodes, shared vminsert gets plain addresses + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}}}, + {Name: "cold", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}}}, + }, + }, + }, + func(t *testing.T, args []string) { + // vmselect: pools are grouped into a single storageNode flag (poolName/addr entries) + var storageNodeArg string + for _, a := range args { + if strings.HasPrefix(a, "-storageNode=") { + storageNodeArg = a + break + } + } + assert.Contains(t, storageNodeArg, "hot/", "vmselect: expected hot pool storageNode, got %v", args) + assert.Contains(t, storageNodeArg, "cold/", "vmselect: expected cold pool storageNode, got %v", args) + // no top-level (ungrouped) storage node when pools are defined + for _, val := range strings.Split(strings.TrimPrefix(storageNodeArg, "-storageNode="), ",") { + assert.True(t, strings.Contains(val, "/"), "vmselect: expected all storageNodes to be pool-grouped, got %q", val) + } + }, + func(t *testing.T, args []string) { + // shared vminsert: pool storage nodes are plain addresses without pool prefix + assert.True(t, hasArg(args, "-storageNode="), "vminsert: expected storageNode flag, got %v", args) + assert.False(t, hasArg(args, "-storageNode=hot/"), "vminsert: should not have pool-grouped addresses, got %v", args) + assert.False(t, hasArg(args, "-storageNode=cold/"), "vminsert: should not have pool-grouped addresses, got %v", args) + }, + ) + + // pools + discovery: srv+ addresses stay pool-grouped, and discoveryInterval/filter + // must still be emitted (previously dropped entirely whenever pools were defined) + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{ + Enabled: true, + Interval: "5s", + Filter: `vmstorage-test-.*\.`, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}}}, + {Name: "cold", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}}}, + }, + }, + }, + func(t *testing.T, args []string) { + var storageNodeArg string + for _, a := range args { + if strings.HasPrefix(a, "-storageNode=") { + storageNodeArg = a + break + } + } + assert.Contains(t, storageNodeArg, "hot/srv+", "vmselect: expected hot pool srv+ storageNode, got %v", args) + assert.Contains(t, storageNodeArg, "cold/srv+", "vmselect: expected cold pool srv+ storageNode, got %v", args) + assert.True(t, hasArg(args, "-storageNode.discoveryInterval=5s"), "vmselect: expected discoveryInterval flag even with pools, got %v", args) + assert.True(t, hasArg(args, `-storageNode.filter=vmstorage-test-.*\.`), "vmselect: expected filter flag even with pools, got %v", args) + }, + nil, + ) + // component-level discovery overrides interval and filter f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, diff --git a/internal/converter/converter.go b/internal/converter/converter.go index 62fc8246ce..ff4efafcb6 100644 --- a/internal/converter/converter.go +++ b/internal/converter/converter.go @@ -17,7 +17,6 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" - "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" ) type VMSingleHelmValues struct { @@ -494,7 +493,7 @@ func MergeValues(base, override []byte) ([]byte, error) { normalizeHeaderMaps(overrideMap) return k8syaml.Marshal(overrideMap) } - if err := build.MergeDeep(&baseMap, &overrideMap, false); err != nil { + if err := vmv1beta1.MergeDeep(&baseMap, &overrideMap, false); err != nil { return nil, fmt.Errorf("cannot merge values: %w", err) } normalizeHeaderMaps(baseMap) diff --git a/internal/podutil/util.go b/internal/podutil/util.go index 6fa0026676..65323d586f 100644 --- a/internal/podutil/util.go +++ b/internal/podutil/util.go @@ -16,7 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmv1alpha1 "github.com/VictoriaMetrics/operator/api/operator/v1alpha1" - "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" ) @@ -284,7 +284,7 @@ func MergeSpecs[T any](a, b *T, name string) (*T, error) { if err != nil { return nil, fmt.Errorf("failed to render spec: %w", err) } - if err := build.MergeDeep(merged, b, false); err != nil { + if err := vmv1beta1.MergeDeep(merged, b, false); err != nil { return nil, fmt.Errorf("failed to merge spec: %w", err) } return merged, nil