From e14d71be58b5dc9ac988539fbb4275bb4afad8a5 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Thu, 27 Aug 2026 20:04:07 +0530 Subject: [PATCH] e2e: add controller recovery test --- test/e2e/bootcnode_test.go | 129 +++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 5c2c330..1bddeed 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/types" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -582,6 +583,134 @@ func TestNonExistingImage(t *testing.T) { t.Logf("Verified node %q did not stage non-existing image", nodeName) } +// TestControllerRecovery provisions a worker node, scales the controller +// deployment to zero, creates a pool with an update image, then scales +// the controller back up and verifies the rollout recovers and completes. +func TestControllerRecovery(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + nodeName := env.AddNode(t) + + ctx := context.Background() + + // Phase 1: Create pool with original image and wait for Idle. + pool := env.NewPool("bnp-recovery", env.NodeImageDigestedPullSpec()) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(3 * time.Minute).Should(And( + HaveField("Booted", Not(BeNil())), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))), + )) + + t.Logf("Node %q is Idle with original image", nodeName) + + // Phase 2: Scale controller deployment to zero. + controllerDeploy := &appsv1.Deployment{} + deployKey := client.ObjectKey{ + Namespace: "bootc-operator", + Name: "bootc-operator-controller-manager", + } + g.Expect(env.Client.Get(ctx, deployKey, controllerDeploy)).To(Succeed()) + + modified := controllerDeploy.DeepCopy() + zero := int32(0) + modified.Spec.Replicas = &zero + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(controllerDeploy))).To(Succeed()) + + g.Eventually(func() (int32, error) { + var d appsv1.Deployment + err := env.Client.Get(ctx, deployKey, &d) + return d.Status.AvailableReplicas, err + }).WithTimeout(1*time.Minute).Should(BeZero(), + "expected controller to scale to zero") + + t.Logf("Controller scaled to zero") + + // Phase 3: Patch pool to update image while controller is down. + updateRef := env.NodeImageUpdateDigestedPullSpec() + + poolModified := pool.DeepCopy() + poolModified.Spec.Image.Ref = updateRef + g.Expect(env.Client.Patch(ctx, poolModified, client.MergeFrom(pool))).To(Succeed()) + *pool = *poolModified + + t.Logf("Patched pool to update image %s while controller is down", updateRef) + + // Wait briefly to confirm nothing happens while controller is down. + g.Consistently(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(10*time.Second).WithPolling(2*time.Second).Should( + HaveField("Booted.ImageDigest", Equal(env.NodeImageDigest())), + "node should not update while controller is down", + ) + + // Phase 4: Scale controller back to one. + g.Expect(env.Client.Get(ctx, deployKey, controllerDeploy)).To(Succeed()) + modified = controllerDeploy.DeepCopy() + one := int32(1) + modified.Spec.Replicas = &one + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(controllerDeploy))).To(Succeed()) + + g.Eventually(func() (int32, error) { + var d appsv1.Deployment + err := env.Client.Get(ctx, deployKey, &d) + return d.Status.AvailableReplicas, err + }).WithTimeout(2*time.Minute).Should(Equal(int32(1)), + "expected controller to scale back to one") + + t.Logf("Controller scaled back to one") + + // Restore replicas on cleanup in case the test fails mid-way. + t.Cleanup(func() { + var d appsv1.Deployment + if err := env.Client.Get(ctx, deployKey, &d); err != nil { + return + } + if d.Spec.Replicas != nil && *d.Spec.Replicas == 0 { + restore := d.DeepCopy() + one := int32(1) + restore.Spec.Replicas = &one + _ = env.Client.Patch(ctx, restore, client.MergeFrom(&d)) + } + }) + + // Phase 5: Wait for rollout to complete after recovery. + g.Eventually(func() (bootcv1alpha1.BootcNodeStatus, error) { + var bn bootcv1alpha1.BootcNode + err := env.Client.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Status, err + }).WithTimeout(5*time.Minute).Should(And( + HaveField("Booted", And( + Not(BeNil()), + HaveField("ImageDigest", env.NodeImageUpdateDigest()), + )), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + ))), + ), "expected node to reach Idle with update image after controller recovery") + + t.Logf("Node %q completed update after controller recovery", nodeName) + + // Verify pool status after recovery. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) +} + func fetchPoolStatus( ctx context.Context, c client.Client,