Add MinReadySeconds to 5 in StatefulSets to prevent sudden double res… (

#647)

* Add Spec.MindReadySeconds to allow setting a non-default value for MinReadySeconds in StatefulSets. Set the default MinReadySeconds to 5 in StatefulSets to prevent sudden double restarts from statefulset controller.

* Modify unit test to ensure value is set

* Add additional test that the change is actually applied

CHANGELOG.md

@@ -11,6 +11,8 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 
 ## unreleased
 
+* [ENHANCEMENT] [#648](https://github.com/k8ssandra/cass-operator/issues/648) Make MinReadySeconds configurable value in the Spec.
+
 ## v1.21.1
 
 * [BUGFIX] [#665](https://github.com/k8ssandra/cass-operator/issues/665) The RequiresUpdate and ObservedGenerations were updated too often, even when the reconcile was not finished.

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -258,6 +258,10 @@ type CassandraDatacenterSpec struct {
 	// Use cautiously.
 	// +optional
 	DatacenterName string `json:"datacenterName,omitempty"`
+
+	// MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec.
+	// Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings.
+	MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
 }
 
 type NetworkingConfig struct {

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -348,6 +348,12 @@ spec:
                     - serverSecretName
                     type: object
                 type: object
+              minReadySeconds:
+                description: |-
+                  MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec.
+                  Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings.
+                format: int32
+                type: integer
               networking:
                 properties:
                   hostNetwork:

pkg/reconciliation/construct_statefulset.go

@@ -145,6 +145,7 @@ func newStatefulSetForCassandraDatacenter(
 			PodManagementPolicy:  appsv1.ParallelPodManagement,
 			Template:             *template,
 			VolumeClaimTemplates: volumeClaimTemplates,
+			MinReadySeconds:      5,
 		},
 	}
 
@@ -169,6 +170,10 @@ func newStatefulSetForCassandraDatacenter(
 		result.Spec.UpdateStrategy = strategy
 	}
 
+	if dc.Spec.MinReadySeconds != nil {
+		result.Spec.MinReadySeconds = *dc.Spec.MinReadySeconds
+	}
+
 	// add a hash here to facilitate checking if updates are needed
 	utils.AddHashAnnotation(result)
 

pkg/reconciliation/construct_statefulset_test.go

@@ -420,6 +420,7 @@ func Test_newStatefulSetForCassandraDatacenterWithAdditionalVolumes(t *testing.T
 		assert.Equal(t, 1, len(got.Spec.Template.Spec.InitContainers[1].VolumeMounts))
 		assert.Equal(t, "server-config", got.Spec.Template.Spec.InitContainers[1].VolumeMounts[0].Name)
 		assert.Equal(t, "/config", got.Spec.Template.Spec.InitContainers[1].VolumeMounts[0].MountPath)
+		assert.Equal(t, int32(5), got.Spec.MinReadySeconds)
 	}
 }
 
@@ -711,3 +712,35 @@ func TestPodTemplateSpecHashAnnotationChanges(t *testing.T) {
 	updatedHash = sts.Annotations[utils.ResourceHashAnnotationKey]
 	assert.NotEqual(currentHash, updatedHash, "expected hash to change when PodTemplateSpec labels change")
 }
+
+func TestMinReadySecondsChange(t *testing.T) {
+	assert := assert.New(t)
+	dc := &api.CassandraDatacenter{
+		Spec: api.CassandraDatacenterSpec{
+			ClusterName:   "test",
+			ServerType:    "cassandra",
+			ServerVersion: "4.0.7",
+			StorageConfig: api.StorageConfig{
+				CassandraDataVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{},
+			},
+			Racks: []api.Rack{
+				{
+					Name: "testrack",
+				},
+			},
+			PodTemplateSpec: &corev1.PodTemplateSpec{},
+		},
+	}
+
+	sts, err := newStatefulSetForCassandraDatacenter(nil, dc.Spec.Racks[0].Name, dc, 3)
+	assert.NoError(err, "failed to build statefulset")
+
+	assert.Equal(int32(5), sts.Spec.MinReadySeconds)
+
+	dc.Spec.MinReadySeconds = ptr.To[int32](10)
+
+	sts, err = newStatefulSetForCassandraDatacenter(nil, dc.Spec.Racks[0].Name, dc, 3)
+	assert.NoError(err, "failed to build statefulset")
+
+	assert.Equal(int32(10), sts.Spec.MinReadySeconds)
+}