PVC resizing support

Signed-off-by: Alexey Makhov <[email protected]>

cmd/main.go

@@ -146,6 +146,7 @@ func main() {
 		Scheme:     mgr.GetScheme(),
 		ClientSet:  clientSet,
 		RESTConfig: restConfig,
+		Recorder:   mgr.GetEventRecorderFor("cluster-reconciler"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "K0smotronCluster")
 		os.Exit(1)

config/rbac/role.yaml

@@ -236,6 +236,7 @@ rules:
   resources:
   - pods
   verbs:
+  - delete
   - get
   - list
 - apiGroups:
@@ -420,6 +421,4 @@ rules:
   verbs:
   - get
   - list
-  - patch
-  - update
   - watch

internal/controller/k0smotron.io/k0smotroncluster_controller.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/secret"
@@ -49,6 +50,7 @@ type ClusterReconciler struct {
 	Scheme     *runtime.Scheme
 	ClientSet  *kubernetes.Clientset
 	RESTConfig *rest.Config
+	Recorder   record.EventRecorder
 }
 
 //+kubebuilder:rbac:groups=k0smotron.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete
@@ -60,10 +62,10 @@ type ClusterReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list
+// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;delete
 // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

internal/controller/k0smotron.io/k0smotroncluster_etcd.go

@@ -130,7 +130,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in
 
 	size := kmc.Spec.Etcd.Persistence.Size
 
-	if n, _ := size.AsInt64(); n == 0 {
+	if size.IsZero() {
 		size = resource.MustParse("1Gi")
 	}
 	pvc := v1.PersistentVolumeClaim{
@@ -299,6 +299,18 @@ func (r *ClusterReconciler) initialCluster(kmc *km.Cluster, replicas int32) stri
 
 func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Container {
 	return []v1.Container{
+		{
+			// Wait for the pods dns name is resolvable, since it takes some tima after the pod is created
+			// and etcd tries to connect to the other members using the dns names
+			Name:            "dns-check",
+			Image:           kmc.Spec.GetImage(),
+			ImagePullPolicy: v1.PullIfNotPresent,
+			Command:         []string{"/bin/bash", "-c"},
+			Args:            []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"},
+			Env: []v1.EnvVar{
+				{Name: "SVC_NAME", Value: kmc.GetEtcdServiceName()},
+			},
+		},
 		{
 			Name:            "init",
 			Image:           kmc.Spec.Etcd.Image,

internal/controller/k0smotron.io/k0smotroncluster_pvc.go

@@ -8,6 +8,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,7 +30,7 @@ func (r *ClusterReconciler) reconcilePVC(ctx context.Context, kmc km.Cluster) er
 	}
 	err = r.reconcileEtcdPVC(ctx, kmc)
 	if err != nil {
-		return fmt.Errorf("failed to reconcile control plane PVC: %w", err)
+		return fmt.Errorf("failed to reconcile etcd PVC: %w", err)
 	}
 
 	return nil
@@ -41,128 +42,80 @@ func (r *ClusterReconciler) reconcileControlPlanePVC(ctx context.Context, kmc km
 		return nil
 	}
 
-	var sts appsv1.StatefulSet
-	err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetStatefulSetName()}, &sts)
-	if err != nil {
-		// Do nothing if StatefulSet does not exist yet
-		if apierrors.IsNotFound(err) {
-			return nil
-		}
-
-		return fmt.Errorf("failed to get statefulset: %w", err)
-	}
-
-	// Do nothing if the sizes match
-	if kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests.Storage().Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
-		return nil
-	}
-
-	// Update the PVC size
-	var allowExpansion *bool
-	for i := 0; i < int(kmc.Spec.Replicas); i++ {
-
-		if kmc.Spec.Persistence.PersistentVolumeClaim.Name == "" {
-			kmc.Spec.Persistence.PersistentVolumeClaim.Name = kmc.GetVolumeName()
-		}
-		name := fmt.Sprintf("%s-%s-%d", kmc.Spec.Persistence.PersistentVolumeClaim.Name, kmc.GetStatefulSetName(), i)
-
-		var pvc corev1.PersistentVolumeClaim
-		err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
-		if err != nil {
-			return fmt.Errorf("failed to get PVC: %w", err)
-		}
-
-		if allowExpansion == nil {
-			var sc storagev1.StorageClass
-			err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
-			if err != nil {
-				return fmt.Errorf("failed to get StorageClass: %w", err)
-			}
-			allowExpansion = sc.AllowVolumeExpansion
-		}
-
-		if allowExpansion != nil && *allowExpansion {
-			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests[corev1.ResourceStorage]
-			err = r.Update(ctx, &pvc)
-			if err != nil {
-				return fmt.Errorf("failed to update PVC: %w", err)
-			}
-
-			// Remove pod to trigger file system resize
-			err = r.Delete(ctx, &corev1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
-					Namespace: kmc.Namespace,
-				},
-			}, &client.DeleteOptions{})
-
-			if err != nil {
-				return fmt.Errorf("failed to delete pod for resizing: %w", err)
-			}
-		} else {
-			break
-		}
+	if kmc.Spec.Persistence.PersistentVolumeClaim.Name == "" {
+		kmc.Spec.Persistence.PersistentVolumeClaim.Name = kmc.GetVolumeName()
 	}
 
-	return r.Delete(ctx, &sts, &client.DeleteOptions{PropagationPolicy: ptr.To(metav1.DeletePropagationOrphan)})
+	return r.resizeStatefulSetAndPVC(ctx, kmc, *kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests.Storage(), kmc.Spec.Replicas, kmc.GetStatefulSetName(), kmc.Spec.Persistence.PersistentVolumeClaim.Name)
 }
 
 func (r *ClusterReconciler) reconcileEtcdPVC(ctx context.Context, kmc km.Cluster) error {
+	return r.resizeStatefulSetAndPVC(ctx, kmc, kmc.Spec.Etcd.Persistence.Size, calculateDesiredReplicas(&kmc), kmc.GetEtcdStatefulSetName(), "etcd-data")
+}
+
+func (r *ClusterReconciler) resizeStatefulSetAndPVC(ctx context.Context, kmc km.Cluster, desiredStorageSize resource.Quantity, replicas int32, stsName, vctName string) error {
 	var sts appsv1.StatefulSet
-	err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetEtcdStatefulSetName()}, &sts)
+	err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: stsName}, &sts)
 	if err != nil {
 		// Do nothing if StatefulSet does not exist yet
 		if apierrors.IsNotFound(err) {
 			return nil
 		}
 
-		return fmt.Errorf("failed to get etcd statefulset: %w", err)
+		return fmt.Errorf("failed to get statefulset: %w", err)
 	}
 
-	// Do nothing if the sizes match
-	if kmc.Spec.Etcd.Persistence.Size.Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
+	if desiredStorageSize.IsZero() ||
+		len(sts.Spec.VolumeClaimTemplates) == 0 ||
+		desiredStorageSize.Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
 		return nil
 	}
 
-	// Update the PVC size
 	var allowExpansion *bool
-	for i := 0; i < int(calculateDesiredReplicas(&kmc)); i++ {
+	for i := 0; i < int(replicas); i++ {
 		var pvc corev1.PersistentVolumeClaim
 
-		name := fmt.Sprintf("etcd-data-%s-%d", kmc.GetEtcdStatefulSetName(), i)
+		name := fmt.Sprintf("%s-%s-%d", vctName, stsName, i)
 		err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
 		if err != nil {
-			return fmt.Errorf("failed to get etcd PVC: %w", err)
+			if apierrors.IsNotFound(err) {
+				// Do nothing if PVC does not exist yet
+				return nil
+			}
+			return fmt.Errorf("failed to get PVC %s: %w", name, err)
 		}
 
 		if allowExpansion == nil {
 			var sc storagev1.StorageClass
 			err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
 			if err != nil {
-				return fmt.Errorf("failed to get etcd StorageClass: %w", err)
+				return fmt.Errorf("failed to get StorageClass %s: %w", *pvc.Spec.StorageClassName, err)
 			}
 			allowExpansion = sc.AllowVolumeExpansion
 		}
 
 		if allowExpansion != nil && *allowExpansion {
-			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Etcd.Persistence.Size
+			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = desiredStorageSize
 			err = r.Update(ctx, &pvc)
 			if err != nil {
-				return fmt.Errorf("failed to update etcd PVC: %w", err)
+				return fmt.Errorf("failed to update PVC %s: %w", pvc.Name, err)
 			}
 
 			// Remove pod to trigger file system resize
 			err = r.Delete(ctx, &corev1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
+					Name:      fmt.Sprintf("%s-%d", stsName, i),
 					Namespace: kmc.Namespace,
 				},
 			}, &client.DeleteOptions{})
 
 			if err != nil {
-				return fmt.Errorf("failed to delete etcd pod for resizing: %w", err)
+				return fmt.Errorf("failed to delete pod '%s' for resizing: %w", fmt.Sprintf("%s-%d", stsName, i), err)
 			}
 		} else {
+			// Do not check other PVCs if expansion is not allowed and just write an event
+			r.Recorder.Eventf(&kmc, corev1.EventTypeWarning, "PVCExpansionNotAllowed", "PVC expansion is not allowed for the storage class %s", *pvc.Spec.StorageClassName)
+
 			break
 		}
 	}

inttest/Makefile

@@ -38,6 +38,8 @@ clean:
 check-scaling-etcd: TIMEOUT=10m
 check-monitoring: TIMEOUT=7m
 check-ha-controlplane: TIMEOUT=7m
+check-pvc: TIMEOUT=15m
+check-pvc: LOCAL_STORAGE_INSTALL_YAML=$(realpath ./footloose-alpine/seaweedfs.yaml)
 check-capi-controlplane-docker-tunneling: TIMEOUT=10m
 check-capi-remote-machine: TIMEOUT=8m
 check-capi-remote-machine-template-update: TIMEOUT=10m