Merge pull request #1785 from stuggi/backup_restore_pause

Add staged deployment support for control plane

Author: openshift-merge-bot[bot]

api/core/v1beta1/conditions.go

@@ -161,6 +161,11 @@ const (
 
 	// OpenStackControlPlaneExposeWatcherReadyCondition Status=True condition which indicates if Watcher is exposed via a route
 	OpenStackControlPlaneExposeWatcherReadyCondition condition.Type = "OpenStackControlPlaneExposeWatcherReady"
+
+	// OpenStackControlPlaneInfrastructureReadyCondition Status=True condition which indicates if infrastructure components are ready
+	// Infrastructure includes: CAs, DNSMasq, RabbitMQ, Galera (MariaDB), Memcached, and OVN databases
+	// This condition is set to True when deployment-stage annotation is "infrastructure-only" and all infrastructure is ready
+	OpenStackControlPlaneInfrastructureReadyCondition condition.Type = "OpenStackControlPlaneInfrastructureReady"
 )
 
 // Common Messages used by API objects.
@@ -507,6 +512,24 @@ const (
 
 	// OpenStackControlPlaneWatcherReadyErrorMessage
 	OpenStackControlPlaneWatcherReadyErrorMessage = "OpenStackControlPlane Watcher error occured %s"
+
+	// OpenStackControlPlaneInfrastructureReadyInitMessage
+	OpenStackControlPlaneInfrastructureReadyInitMessage = "OpenStackControlPlane Infrastructure not started"
+
+	// OpenStackControlPlaneInfrastructureReadyMessage
+	OpenStackControlPlaneInfrastructureReadyMessage = "OpenStackControlPlane Infrastructure ready"
+
+	// OpenStackControlPlaneInfrastructureReadyRunningMessage
+	OpenStackControlPlaneInfrastructureReadyRunningMessage = "OpenStackControlPlane Infrastructure in progress"
+
+	// OpenStackControlPlaneInfrastructureReadyWaitingMessage
+	OpenStackControlPlaneInfrastructureReadyWaitingMessage = "OpenStackControlPlane Infrastructure in progress - waiting for: %s"
+
+	// OpenStackControlPlaneInfrastructureReadyPausedMessage
+	OpenStackControlPlaneInfrastructureReadyPausedMessage = "OpenStackControlPlane Infrastructure ready - deployment paused. Remove annotation to resume reconcile of OpenStack services"
+
+	// OpenStackControlPlaneInfrastructureReadyErrorMessage
+	OpenStackControlPlaneInfrastructureReadyErrorMessage = "OpenStackControlPlane Infrastructure error occured %s"
 )
 
 // Version Conditions used by to drive minor updates

api/core/v1beta1/openstackcontrolplane_types.go

@@ -66,6 +66,11 @@ const (
 	GlanceName = "glance"
 	// CinderName - Default Cinder name
 	CinderName = "cinder"
+
+	// DeploymentStageAnnotation - Annotation key for controlling deployment stages
+	DeploymentStageAnnotation = "core.openstack.org/deployment-stage"
+	// DeploymentStageInfrastructureOnly - Annotation value to pause after infrastructure deployment
+	DeploymentStageInfrastructureOnly = "infrastructure-only"
 )
 
 // OpenStackControlPlaneSpec defines the desired state of OpenStackControlPlane
@@ -951,6 +956,7 @@ func (instance *OpenStackControlPlane) InitConditions() {
 		condition.UnknownCondition(OpenStackControlPlaneCAReadyCondition, condition.InitReason, OpenStackControlPlaneCAReadyInitMessage),
 		condition.UnknownCondition(OpenStackControlPlaneOpenStackVersionInitializationReadyCondition, condition.InitReason, OpenStackControlPlaneOpenStackVersionInitializationReadyInitMessage),
 		condition.UnknownCondition(OpenStackControlPlaneWatcherReadyCondition, condition.InitReason, OpenStackControlPlaneWatcherReadyInitMessage),
+		condition.UnknownCondition(OpenStackControlPlaneInfrastructureReadyCondition, condition.InitReason, OpenStackControlPlaneInfrastructureReadyInitMessage),
 
 		// Also add the overall status condition as Unknown
 		condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage),

internal/controller/core/openstackcontrolplane_controller.go

@@ -20,6 +20,7 @@ package core
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -186,9 +187,23 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr
 			// something is not ready so reset the Ready condition
 			instance.Status.Conditions.MarkUnknown(
 				condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage)
-			// and recalculate it based on the state of the rest of the conditions
-			instance.Status.Conditions.Set(
-				instance.Status.Conditions.Mirror(condition.ReadyCondition))
+
+			// In infrastructure-only mode with infrastructure ready, set Ready to False with pause message
+			// This prevents service conditions (which are still Unknown/Init) from being mirrored to Ready
+			if stage, ok := instance.Annotations[corev1beta1.DeploymentStageAnnotation]; ok &&
+				stage == corev1beta1.DeploymentStageInfrastructureOnly &&
+				instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneInfrastructureReadyCondition) {
+				// Set Ready to False with the infrastructure pause message
+				instance.Status.Conditions.Set(condition.FalseCondition(
+					condition.ReadyCondition,
+					condition.RequestedReason,
+					condition.SeverityInfo,
+					corev1beta1.OpenStackControlPlaneInfrastructureReadyPausedMessage))
+			} else {
+				// Normal mode or infrastructure not ready yet: use default mirror behavior
+				instance.Status.Conditions.Set(
+					instance.Status.Conditions.Mirror(condition.ReadyCondition))
+			}
 		}
 
 		condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions)
@@ -386,6 +401,36 @@ func (r *OpenStackControlPlaneReconciler) reconcileOVNControllers(ctx context.Co
 	return ctrl.Result{}, nil
 }
 
+// isInfrastructureReady checks if all enabled infrastructure components are ready
+// Returns true if ready, and a list of components that are not ready (empty if all ready)
+func isInfrastructureReady(instance *corev1beta1.OpenStackControlPlane) (bool, []string) {
+	notReady := []string{}
+
+	// CAs are always deployed (no enabled flag)
+	if !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneCAReadyCondition) {
+		notReady = append(notReady, "CAs")
+	}
+
+	// Only check each infrastructure component if it's enabled
+	if instance.Spec.DNS.Enabled && !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneDNSReadyCondition) {
+		notReady = append(notReady, "DNS")
+	}
+	if instance.Spec.Rabbitmq.Enabled && !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneRabbitMQReadyCondition) {
+		notReady = append(notReady, "RabbitMQs")
+	}
+	if instance.Spec.Galera.Enabled && !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMariaDBReadyCondition) {
+		notReady = append(notReady, "Galeras")
+	}
+	if instance.Spec.Memcached.Enabled && !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneMemcachedReadyCondition) {
+		notReady = append(notReady, "Memcached")
+	}
+	if instance.Spec.Ovn.Enabled && !instance.Status.Conditions.IsTrue(corev1beta1.OpenStackControlPlaneOVNReadyCondition) {
+		notReady = append(notReady, "OVN")
+	}
+
+	return len(notReady) == 0, notReady
+}
+
 func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, instance *corev1beta1.OpenStackControlPlane, version *corev1beta1.OpenStackVersion, helper *common_helper.Helper) (ctrl.Result, error) {
 	if instance.Spec.TopologyRef != nil {
 		if err := r.checkTopologyRef(ctx, helper,
@@ -402,6 +447,11 @@ func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, i
 		instance.Status.Conditions.MarkTrue(condition.TopologyReadyCondition, condition.TopologyReadyMessage)
 	}
 
+	// Check for deployment-stage annotation
+	deploymentStage := instance.Annotations[corev1beta1.DeploymentStageAnnotation]
+	infrastructureOnly := deploymentStage == corev1beta1.DeploymentStageInfrastructureOnly
+
+	// Reconcile infrastructure components (always run)
 	ctrlResult, err := openstack.ReconcileCAs(ctx, instance, helper)
 	if err != nil {
 		return ctrl.Result{}, err
@@ -437,41 +487,87 @@ func (r *OpenStackControlPlaneReconciler) reconcileNormal(ctx context.Context, i
 		return ctrlResult, nil
 	}
 
-	ctrlResult, err = openstack.ReconcileKeystoneAPI(ctx, instance, version, helper)
+	// OVN databases are part of infrastructure
+	ctrlResult, err = openstack.ReconcileOVN(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		return ctrlResult, nil
 	}
 
-	ctrlResult, err = openstack.ReconcilePlacementAPI(ctx, instance, version, helper)
+	// Update InfrastructureReady condition based on infrastructure component readiness
+	// This is useful for observability regardless of staged deployment mode
+	infrastructureReady, notReadyComponents := isInfrastructureReady(instance)
+
+	if infrastructureReady {
+		// Set different messages based on whether deployment is paused
+		if infrastructureOnly {
+			// Infrastructure-only mode: indicate deployment is paused
+			instance.Status.Conditions.MarkTrue(
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyCondition,
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyPausedMessage)
+			r.GetLogger(ctx).Info("Infrastructure components ready - deployment paused at infrastructure-only stage")
+		} else {
+			// Normal mode: infrastructure is ready
+			instance.Status.Conditions.MarkTrue(
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyCondition,
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyMessage)
+		}
+	} else {
+		// Build a descriptive message showing which components are not ready
+		if len(notReadyComponents) > 0 {
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyCondition,
+				condition.RequestedReason,
+				condition.SeverityInfo,
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyWaitingMessage,
+				strings.Join(notReadyComponents, ", ")))
+		} else {
+			instance.Status.Conditions.Set(condition.FalseCondition(
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyCondition,
+				condition.RequestedReason,
+				condition.SeverityInfo,
+				corev1beta1.OpenStackControlPlaneInfrastructureReadyRunningMessage))
+		}
+	}
+
+	// Check if we're in infrastructure-only mode and should pause deployment
+	if infrastructureOnly {
+		// Stop here - do not reconcile OpenStack services
+		return ctrl.Result{}, nil
+	}
+
+	// Continue with OpenStack service reconciliation
+	ctrlResult, err = openstack.ReconcileKeystoneAPI(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		return ctrlResult, nil
 	}
 
-	ctrlResult, err = openstack.ReconcileGlance(ctx, instance, version, helper)
+	ctrlResult, err = openstack.ReconcilePlacementAPI(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		return ctrlResult, nil
 	}
 
-	ctrlResult, err = openstack.ReconcileCinder(ctx, instance, version, helper)
+	ctrlResult, err = openstack.ReconcileGlance(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		return ctrlResult, nil
 	}
 
-	ctrlResult, err = openstack.ReconcileOVN(ctx, instance, version, helper)
+	ctrlResult, err = openstack.ReconcileCinder(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err
 	} else if (ctrlResult != ctrl.Result{}) {
 		return ctrlResult, nil
 	}
 
+	// OVN already reconciled in infrastructure section above
+
 	ctrlResult, err = openstack.ReconcileNeutron(ctx, instance, version, helper)
 	if err != nil {
 		return ctrl.Result{}, err

test/kuttl/common/assert-sample-deployment.yaml

@@ -254,6 +254,10 @@ status:
     reason: Ready
     status: "True"
     type: OpenStackControlPlaneGlanceReady
+  - message: OpenStackControlPlane Infrastructure ready
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneInfrastructureReady
   - message: OpenStackControlPlane InstanceHa CM is available
     reason: Ready
     status: "True"

test/kuttl/tests/ctlplane-basic-deployment/03-assert-deploy-custom-cacert.yaml

@@ -63,6 +63,10 @@ status:
     reason: Ready
     status: "True"
     type: OpenStackControlPlaneGlanceReady
+  - message: OpenStackControlPlane Infrastructure ready
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneInfrastructureReady
   - message: OpenStackControlPlane InstanceHa CM is available
     reason: Ready
     status: "True"

test/kuttl/tests/ctlplane-collapsed/01-assert-collapsed-cell.yaml

@@ -231,6 +231,10 @@ status:
     reason: Ready
     status: "True"
     type: OpenStackControlPlaneGlanceReady
+  - message: OpenStackControlPlane Infrastructure ready
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneInfrastructureReady
   - message: OpenStackControlPlane InstanceHa CM is available
     reason: Ready
     status: "True"

test/kuttl/tests/ctlplane-galera-3replicas/01-assert-galera-3replicas.yaml

@@ -219,6 +219,10 @@ status:
     reason: Ready
     status: "True"
     type: OpenStackControlPlaneGlanceReady
+  - message: OpenStackControlPlane Infrastructure ready
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneInfrastructureReady
   - message: OpenStackControlPlane InstanceHa CM is available
     reason: Ready
     status: "True"

test/kuttl/tests/ctlplane-staged-deployment/01-assert-infrastructure-ready-paused.yaml

@@ -0,0 +1,101 @@
+# Assert that infrastructure is ready but deployment is paused
+# OpenStackControlPlaneInfrastructureReady should be True with pause message
+# Infrastructure components (CAs, RabbitMQ, MariaDB, Memcached, OVN) should be Ready
+# Service components should still be in Init/Unknown state
+# Overall Ready should be False
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+  annotations:
+    core.openstack.org/deployment-stage: infrastructure-only
+status:
+  conditions:
+  # Overall deployment should not be ready yet (paused after infrastructure)
+  - message: OpenStackControlPlane Infrastructure ready - deployment paused. Remove annotation to resume reconcile of OpenStack services
+    reason: Requested
+    status: "False"
+    type: Ready
+  # Following conditions are alphabetically sorted by type
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneBarbicanReady
+  - message: OpenStackControlPlane CAs completed
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneCAReadyCondition
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneCinderReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneClientReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneDesignateReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneGlanceReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneHeatReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneHorizonReady
+  - message: OpenStackControlPlane Infrastructure ready - deployment paused. Remove annotation to resume reconcile of OpenStack services
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneInfrastructureReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneIronicReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneKeystoneAPIReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneManilaReady
+  - message: OpenStackControlPlane MariaDB completed
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneMariaDBReady
+  - message: OpenStackControlPlane Memcached completed
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneMemcachedReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneNeutronReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneNovaReady
+  - message: OpenStackControlPlane OVN completed
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneOVNReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneOctaviaReady
+  - message: OpenStackControlPlane OpenStackVersion initialized
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneOpenStackVersionInitializationReadyCondition
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlanePlacementAPIReady
+  - message: OpenStackControlPlane RabbitMQ completed
+    reason: Ready
+    status: "True"
+    type: OpenStackControlPlaneRabbitMQReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneRedisReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneSwiftReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneTelemetryReady
+  - reason: Init
+    status: Unknown
+    type: OpenStackControlPlaneWatcherReady

test/kuttl/tests/ctlplane-staged-deployment/01-deploy-openstack-infrastructure-only.yaml

@@ -0,0 +1,10 @@
+# Deploy OpenStackControlPlane with deployment-stage annotation set to infrastructure-only
+# This should pause deployment after infrastructure (CAs, RabbitMQ, Galera, Memcached, OVN) is ready
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc kustomize ../../../../config/samples/base/openstackcontrolplane | \
+      oc annotate -f - --local=true --dry-run=none \
+        core.openstack.org/deployment-stage=infrastructure-only -o yaml | \
+      oc apply -n $NAMESPACE -f -

test/kuttl/tests/ctlplane-staged-deployment/02-assert-controlplane-ready.yaml

@@ -0,0 +1 @@
+../../common/assert-sample-deployment.yaml