Cleanup legacy installPlan and Operator resource

The operator resource's have references to:
 -CSVs
 -Subscriptions
 -Installplans
 -CRDs

So the cleanup for operators needs to run after applyCRDs
where 'olm.managed' gets removed.

Author: dprince

config/operator/rbac/role.yaml

@@ -107,13 +107,8 @@ rules:
   - operators.coreos.com
   resources:
   - clusterserviceversions
-  verbs:
-  - delete
-  - get
-  - list
-- apiGroups:
-  - operators.coreos.com
-  resources:
+  - installplans
+  - operators
   - subscriptions
   verbs:
   - delete

controllers/operator/openstack_controller.go

@@ -59,21 +59,9 @@ type OpenStackReconciler struct {
 	Kclient kubernetes.Interface
 }
 
-var csvGVR = schema.GroupVersionResource{
-	Group:    "operators.coreos.com",
-	Version:  "v1alpha1",
-	Resource: "clusterserviceversions",
-}
-
-var subscriptionGVR = schema.GroupVersionResource{
-	Group:    "operators.coreos.com",
-	Version:  "v1alpha1",
-	Resource: "subscriptions",
-}
-
 // GetLog returns a logger object with a prefix of "controller.name" and aditional controller context fields
 func (r *OpenStackReconciler) GetLogger(ctx context.Context) logr.Logger {
-	return log.FromContext(ctx).WithName("Controllers").WithName("OpenStackControlPlane")
+	return log.FromContext(ctx).WithName("Controllers").WithName("OpenStackOperator")
 }
 
 var (
@@ -125,8 +113,7 @@ func SetupEnv() {
 // +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch;create;update;patch;delete;
 // +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete;
 // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=list;get;watch;update;create
-// +kubebuilder:rbac:groups=operators.coreos.com,resources=clusterserviceversions,verbs=get;list;delete;
-// +kubebuilder:rbac:groups=operators.coreos.com,resources=subscriptions,verbs=get;list;delete;
+// +kubebuilder:rbac:groups=operators.coreos.com,resources=clusterserviceversions;subscriptions;installplans;operators,verbs=get;list;delete;
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -245,7 +232,7 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return r.reconcileDelete(ctx, instance, openstackHelper)
 	}
 
-	// TODO: cleanup obsolete resources here (remove old CSVs, etc)
+	// cleanup obsolete resources here (remove old CSVs, etc)
 	if err := r.cleanupObsoleteResources(ctx, instance); err != nil {
 		return ctrl.Result{}, err
 	}
@@ -260,6 +247,12 @@ func (r *OpenStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		return ctrl.Result{}, err
 	}
 
+	// now that CRDs have been updated (with old olm.managed references removed)
+	// we can finally cleanup the old operators
+	if err := r.postCleanupObsoleteResources(ctx, instance); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// Check if all deployments are running
 	deploymentsRunning, err := r.countDeployments(ctx, instance)
 	instance.Status.DeployedOperatorCount = &deploymentsRunning
@@ -437,11 +430,14 @@ func (r *OpenStackReconciler) renderAndApply(
 	return nil
 }
 
-var serviceOperatorNames = []string{"barbican", "cinder", "designate", "glance", "heat", "horizon", "infra",
-	"ironic", "keystone", "manila", "mariadb", "neutron", "nova", "octavia", "openstack-baremetal", "ovn",
-	"placement", "rabbitmq-cluster", "swift", "telemetry"}
-
 func isServiceOperatorResource(name string) bool {
+	//NOTE: test-operator was deployed as a independant package so it may or may not be installed
+	//NOTE: depending on how watcher-operator is released for FR2 and then in FR3 it may need to be
+	// added into this list in the future
+	serviceOperatorNames := []string{"barbican", "cinder", "designate", "glance", "heat", "horizon", "infra",
+		"ironic", "keystone", "manila", "mariadb", "neutron", "nova", "octavia", "openstack-baremetal", "ovn",
+		"placement", "rabbitmq-cluster", "swift", "telemetry", "test"}
+
 	for _, item := range serviceOperatorNames {
 		if strings.Index(name, item) == 0 {
 			return true
@@ -450,48 +446,126 @@ func isServiceOperatorResource(name string) bool {
 	return false
 }
 
-// cleanupObsoleteResources - deletes CSVs for old service operator bundles
+// cleanupObsoleteResources - deletes CSVs and subscriptions
 func (r *OpenStackReconciler) cleanupObsoleteResources(ctx context.Context, instance *operatorv1beta1.OpenStack) error {
 	Log := r.GetLogger(ctx)
 
+	csvGVR := schema.GroupVersionResource{
+		Group:    "operators.coreos.com",
+		Version:  "v1alpha1",
+		Resource: "clusterserviceversions",
+	}
+
+	subscriptionGVR := schema.GroupVersionResource{
+		Group:    "operators.coreos.com",
+		Version:  "v1alpha1",
+		Resource: "subscriptions",
+	}
+
+	installPlanGVR := schema.GroupVersionResource{
+		Group:    "operators.coreos.com",
+		Version:  "v1alpha1",
+		Resource: "installplans",
+	}
+
+	csvList := &uns.UnstructuredList{}
+	csvList.SetGroupVersionKind(csvGVR.GroupVersion().WithKind("ClusterServiceVersion"))
+	err := r.Client.List(ctx, csvList, &client.ListOptions{Namespace: instance.Namespace})
+	if err != nil {
+		return err
+	}
+	for _, csv := range csvList.Items {
+		Log.Info("Found CSV", "name", csv.GetName())
+		if isServiceOperatorResource(csv.GetName()) {
+			err = r.Client.Delete(ctx, &csv)
+			if err != nil {
+				return err
+			}
+			Log.Info("CSV deleted successfully", "name", csv.GetName())
+		}
+	}
+
 	subscriptionList := &uns.UnstructuredList{}
 	subscriptionList.SetGroupVersionKind(subscriptionGVR.GroupVersion().WithKind("Subscription"))
-	err := r.Client.List(ctx, subscriptionList, &client.ListOptions{Namespace: instance.Namespace})
+	err = r.Client.List(ctx, subscriptionList, &client.ListOptions{Namespace: instance.Namespace})
 	if err != nil {
-		Log.Error(err, "Unable to retrieve Subscriptions instances")
 		return err
 	}
 	for _, subscription := range subscriptionList.Items {
 		Log.Info("Found Subscription", "name", subscription.GetName())
 		if isServiceOperatorResource(subscription.GetName()) {
 			err = r.Client.Delete(ctx, &subscription)
 			if err != nil {
-				Log.Error(err, "Failed to delete existing Subscription")
 				return err
 			}
+			Log.Info("Subscription deleted successfully", "name", subscription.GetName())
 		}
-		Log.Info("Subscription deleted successfully")
 	}
 
-	csvList := &uns.UnstructuredList{}
-	csvList.SetGroupVersionKind(csvGVR.GroupVersion().WithKind("ClusterServiceVersionList"))
+	// lookup the installplan which has the clusterServiceVersionNames we removed above
+	// there will be just a single installPlan that has all of them referenced
+	installPlanList := &uns.UnstructuredList{}
+	installPlanList.SetGroupVersionKind(installPlanGVR.GroupVersion().WithKind("InstallPlan"))
 
-	err = r.Client.List(ctx, csvList, &client.ListOptions{Namespace: instance.Namespace})
+	err = r.Client.List(ctx, installPlanList, &client.ListOptions{Namespace: instance.Namespace})
 	if err != nil {
-		Log.Error(err, "Unable to retrieve CSV instances")
 		return err
 	}
-	for _, csv := range csvList.Items {
-		Log.Info("Found CSV", "name", csv.GetName())
-		if isServiceOperatorResource(csv.GetName()) {
-			err = r.Client.Delete(ctx, &csv)
+	for _, installPlan := range installPlanList.Items {
+		Log.Info("Found installPlan", "name", installPlan.GetName())
+		// this should have a list containing the CSV names of all the old/legacy service operator CSVs
+		csvNames, found, err := uns.NestedSlice(installPlan.Object, "spec", "clusterServiceVersionNames")
+		if err != nil {
+			return err
+		}
+		if found {
+			// just checking for the first one should be sufficient
+			if isServiceOperatorResource(csvNames[0].(string)) {
+				err = r.Client.Delete(ctx, &installPlan)
+				if err != nil {
+					return err
+				}
+				Log.Info("Installplan deleted successfully", "name", installPlan.GetName())
+			}
+		}
+	}
+
+	return nil
+
+}
+
+// postCleanupObsoleteResources - deletes CSVs for old service operator bundles
+func (r *OpenStackReconciler) postCleanupObsoleteResources(ctx context.Context, instance *operatorv1beta1.OpenStack) error {
+	Log := r.GetLogger(ctx)
+
+	operatorGVR := schema.GroupVersionResource{
+		Group:    "operators.coreos.com",
+		Version:  "v1",
+		Resource: "operators",
+	}
+
+	// finally we can remove operator objects as all the refs have been cleaned up:
+	// 1) CSVs
+	// 2) Subscriptions
+	// 3) CRD olm.managed references removed
+	// 4) installPlan from old service operators removed
+	operatorList := &uns.UnstructuredList{}
+	operatorList.SetGroupVersionKind(operatorGVR.GroupVersion().WithKind("Operator"))
+	err := r.Client.List(ctx, operatorList, &client.ListOptions{Namespace: instance.Namespace})
+	if err != nil {
+		return err
+	}
+	for _, operator := range operatorList.Items {
+		Log.Info("Found Operator", "name", operator.GetName())
+		if isServiceOperatorResource(operator.GetName()) {
+			err = r.Client.Delete(ctx, &operator)
 			if err != nil {
-				Log.Error(err, "Failed to delete existing CSV")
 				return err
 			}
+			Log.Info("Operator deleted successfully", "name", operator.GetName())
 		}
-		Log.Info("CSV deleted successfully")
 	}
+
 	return nil
 
 }