add IT for bound binding being evicted

pkg/controllers/rollout/controller_integration_test.go

@@ -657,6 +657,7 @@ var _ = Describe("Test the rollout Controller", func() {
 		newScheduledBinding := generateClusterResourceBinding(fleetv1beta1.BindingStateScheduled, masterSnapshot.Name, cluster3)
 		Expect(k8sClient.Create(ctx, newScheduledBinding)).Should(Succeed())
 		By(fmt.Sprintf("resource binding  %s created", newScheduledBinding.Name))
+		// add new scheduled binding to list of bindings.
 		bindings = append(bindings, newScheduledBinding)
 
 		// ensure new binding exists.
@@ -715,6 +716,93 @@ var _ = Describe("Test the rollout Controller", func() {
 		}
 	})
 
+	It("Evict bound binding", func() {
+		// create CRP
+		var targetCluster int32 = 2
+		rolloutCRP = clusterResourcePlacementForTest(testCRPName, createPlacementPolicyForTest(fleetv1beta1.PickNPlacementType, targetCluster))
+		// Set MaxSurge to 0.
+		rolloutCRP.Spec.Strategy.RollingUpdate.MaxSurge = &intstr.IntOrString{
+			Type:   intstr.Int,
+			IntVal: 0,
+		}
+		Expect(k8sClient.Create(ctx, rolloutCRP)).Should(Succeed())
+		// create master resource snapshot that is latest.
+		masterSnapshot := generateResourceSnapshot(rolloutCRP.Name, 0, true)
+		Expect(k8sClient.Create(ctx, masterSnapshot)).Should(Succeed())
+
+		// create scheduled bindings for master snapshot on target clusters
+		clusters := make([]string, targetCluster)
+		for i := 0; i < int(targetCluster); i++ {
+			clusters[i] = "cluster-" + utils.RandStr()
+			binding := generateClusterResourceBinding(fleetv1beta1.BindingStateScheduled, masterSnapshot.Name, clusters[i])
+			Expect(k8sClient.Create(ctx, binding)).Should(Succeed())
+			By(fmt.Sprintf("resource binding  %s created", binding.Name))
+			bindings = append(bindings, binding)
+		}
+
+		// check that all bindings are bound.
+		Eventually(func() bool {
+			for _, binding := range bindings {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: binding.GetName()}, binding)
+				if err != nil {
+					return false
+				}
+				if binding.Spec.State != fleetv1beta1.BindingStateBound || binding.Spec.ResourceSnapshotName != masterSnapshot.Name {
+					return false
+				}
+			}
+			return true
+		}, timeout, interval).Should(BeTrue(), "rollout controller should roll all the bindings to Bound state")
+
+		// Note: This scenario is very unlikely in production user has to change the target from 2->3->2,
+		// where scheduled created new scheduled binding but user changed the target number to 2 again, before rollout controller reads CRP.
+		// create a new scheduled binding.
+		cluster3 = "cluster-" + utils.RandStr()
+		newScheduledBinding := generateClusterResourceBinding(fleetv1beta1.BindingStateScheduled, masterSnapshot.Name, cluster3)
+		Expect(k8sClient.Create(ctx, newScheduledBinding)).Should(Succeed())
+		By(fmt.Sprintf("resource binding  %s created", newScheduledBinding.Name))
+		// add new scheduled binding to list of bindings.
+		bindings = append(bindings, newScheduledBinding)
+
+		// ensure new binding exists.
+		Eventually(func() bool {
+			return !apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: newScheduledBinding.Name}, newScheduledBinding))
+		}, timeout, interval).Should(BeTrue(), "new scheduled binding is not found")
+
+		// Current state: rollout is blocked by maxSurge being 0.
+		// check if new scheduled binding is not bound.
+		Consistently(func() error {
+			err := k8sClient.Get(ctx, types.NamespacedName{Name: newScheduledBinding.Name}, newScheduledBinding)
+			if err != nil {
+				return err
+			}
+			if newScheduledBinding.Spec.State == fleetv1beta1.BindingStateBound {
+				return fmt.Errorf("binding %s is in bound state, which is unexpected", newScheduledBinding.Name)
+			}
+			return nil
+		}, timeout, interval).Should(BeNil(), "rollout controller shouldn't roll new scheduled binding to bound state")
+
+		// simulate eviction by deleting first bound binding.
+		firstBoundBinding := 1
+		for i := 0; i < firstBoundBinding; i++ {
+			Expect(k8sClient.Delete(ctx, bindings[i])).Should(Succeed())
+		}
+
+		// check to see if the remaining two bindings are bound.
+		for i := firstBoundBinding; i < len(bindings); i++ {
+			Eventually(func() bool {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: bindings[i].GetName()}, bindings[i])
+				if err != nil {
+					return false
+				}
+				if bindings[i].Spec.State != fleetv1beta1.BindingStateBound || bindings[i].Spec.ResourceSnapshotName != masterSnapshot.Name {
+					return false
+				}
+				return true
+			}, timeout, interval).Should(BeTrue(), "rollout controller should roll all remaining bindings to Bound state")
+		}
+	})
+
 	// TODO: should update scheduled bindings to the latest snapshot when it is updated to bound state.
 
 	// TODO: should count the deleting bindings as can be Unavailable.