Skip to content

Commit

Permalink
PVC resizing support
Browse files Browse the repository at this point in the history
Signed-off-by: Alexey Makhov <[email protected]>
  • Loading branch information
makhov committed Jun 13, 2024
1 parent 3ca59d4 commit 90e5907
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 78 deletions.
12 changes: 11 additions & 1 deletion internal/controller/k0smotron.io/k0smotroncluster_etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in

size := kmc.Spec.Etcd.Persistence.Size

if n, _ := size.AsInt64(); n == 0 {
if size.IsZero() {
size = resource.MustParse("1Gi")
}
pvc := v1.PersistentVolumeClaim{
Expand Down Expand Up @@ -299,6 +299,16 @@ func (r *ClusterReconciler) initialCluster(kmc *km.Cluster, replicas int32) stri

func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Container {
return []v1.Container{
{
Name: "dns-check",
Image: kmc.Spec.GetImage(),
ImagePullPolicy: v1.PullIfNotPresent,
Command: []string{"/bin/bash", "-c"},
Args: []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"},
Env: []v1.EnvVar{
{Name: "SVC_NAME", Value: kmc.GetEtcdServiceName()},
},
},
{
Name: "init",
Image: kmc.Spec.Etcd.Image,
Expand Down
103 changes: 27 additions & 76 deletions internal/controller/k0smotron.io/k0smotroncluster_pvc.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package k0smotronio
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/api/resource"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -29,7 +30,7 @@ func (r *ClusterReconciler) reconcilePVC(ctx context.Context, kmc km.Cluster) er
}
err = r.reconcileEtcdPVC(ctx, kmc)
if err != nil {
return fmt.Errorf("failed to reconcile control plane PVC: %w", err)
return fmt.Errorf("failed to reconcile etcd PVC: %w", err)
}

return nil
Expand All @@ -41,128 +42,78 @@ func (r *ClusterReconciler) reconcileControlPlanePVC(ctx context.Context, kmc km
return nil
}

var sts appsv1.StatefulSet
err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetStatefulSetName()}, &sts)
if err != nil {
// Do nothing if StatefulSet does not exist yet
if apierrors.IsNotFound(err) {
return nil
}

return fmt.Errorf("failed to get statefulset: %w", err)
}

// Do nothing if the sizes match
if kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests.Storage().Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
return nil
if kmc.Spec.Persistence.PersistentVolumeClaim.Name == "" {
kmc.Spec.Persistence.PersistentVolumeClaim.Name = kmc.GetVolumeName()
}

// Update the PVC size
var allowExpansion *bool
for i := 0; i < int(kmc.Spec.Replicas); i++ {

if kmc.Spec.Persistence.PersistentVolumeClaim.Name == "" {
kmc.Spec.Persistence.PersistentVolumeClaim.Name = kmc.GetVolumeName()
}
name := fmt.Sprintf("%s-%s-%d", kmc.Spec.Persistence.PersistentVolumeClaim.Name, kmc.GetStatefulSetName(), i)

var pvc corev1.PersistentVolumeClaim
err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
if err != nil {
return fmt.Errorf("failed to get PVC: %w", err)
}

if allowExpansion == nil {
var sc storagev1.StorageClass
err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
if err != nil {
return fmt.Errorf("failed to get StorageClass: %w", err)
}
allowExpansion = sc.AllowVolumeExpansion
}

if allowExpansion != nil && *allowExpansion {
pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests[corev1.ResourceStorage]
err = r.Update(ctx, &pvc)
if err != nil {
return fmt.Errorf("failed to update PVC: %w", err)
}

// Remove pod to trigger file system resize
err = r.Delete(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
Namespace: kmc.Namespace,
},
}, &client.DeleteOptions{})

if err != nil {
return fmt.Errorf("failed to delete pod for resizing: %w", err)
}
} else {
break
}
}

return r.Delete(ctx, &sts, &client.DeleteOptions{PropagationPolicy: ptr.To(metav1.DeletePropagationOrphan)})
return r.resizeStatefulSetAndPVC(ctx, kmc, *kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests.Storage(), kmc.Spec.Replicas, kmc.GetStatefulSetName(), kmc.Spec.Persistence.PersistentVolumeClaim.Name)
}

func (r *ClusterReconciler) reconcileEtcdPVC(ctx context.Context, kmc km.Cluster) error {
return r.resizeStatefulSetAndPVC(ctx, kmc, kmc.Spec.Etcd.Persistence.Size, calculateDesiredReplicas(&kmc), kmc.GetEtcdStatefulSetName(), "etcd-data")
}

func (r *ClusterReconciler) resizeStatefulSetAndPVC(ctx context.Context, kmc km.Cluster, desiredStorageSize resource.Quantity, replicas int32, stsName, vctName string) error {
var sts appsv1.StatefulSet
err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetEtcdStatefulSetName()}, &sts)
err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetStatefulSetName()}, &sts)
if err != nil {
// Do nothing if StatefulSet does not exist yet
if apierrors.IsNotFound(err) {
return nil
}

return fmt.Errorf("failed to get etcd statefulset: %w", err)
return fmt.Errorf("failed to get statefulset: %w", err)
}

// Do nothing if the sizes match
if kmc.Spec.Etcd.Persistence.Size.Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
if desiredStorageSize.IsZero() ||
len(sts.Spec.VolumeClaimTemplates) == 0 ||
desiredStorageSize.Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
return nil
}

// Update the PVC size
var allowExpansion *bool
for i := 0; i < int(calculateDesiredReplicas(&kmc)); i++ {
for i := 0; i < int(replicas); i++ {
var pvc corev1.PersistentVolumeClaim

name := fmt.Sprintf("etcd-data-%s-%d", kmc.GetEtcdStatefulSetName(), i)
name := fmt.Sprintf("%s-%s-%d", vctName, stsName, i)
err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
if err != nil {
return fmt.Errorf("failed to get etcd PVC: %w", err)
if apierrors.IsNotFound(err) {
// Do nothing if PVC does not exist yet
return nil
}
return fmt.Errorf("failed to get PVC %s: %w", name, err)
}

if allowExpansion == nil {
var sc storagev1.StorageClass
err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
if err != nil {
return fmt.Errorf("failed to get etcd StorageClass: %w", err)
return fmt.Errorf("failed to get StorageClass %s: %w", *pvc.Spec.StorageClassName, err)
}
allowExpansion = sc.AllowVolumeExpansion
}

if allowExpansion != nil && *allowExpansion {
pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Etcd.Persistence.Size
pvc.Spec.Resources.Requests[corev1.ResourceStorage] = desiredStorageSize
err = r.Update(ctx, &pvc)
if err != nil {
return fmt.Errorf("failed to update etcd PVC: %w", err)
return fmt.Errorf("failed to update PVC %s: %w", pvc.Name, err)
}

// Remove pod to trigger file system resize
err = r.Delete(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
Name: fmt.Sprintf("%s-%d", stsName, i),
Namespace: kmc.Namespace,
},
}, &client.DeleteOptions{})

if err != nil {
return fmt.Errorf("failed to delete etcd pod for resizing: %w", err)
return fmt.Errorf("failed to delete pod '%s' for resizing: %w", fmt.Sprintf("%s-%d", stsName, i), err)
}
} else {
// Do not check other PVCs if expansion is not allowed
break
}
}
Expand Down
2 changes: 1 addition & 1 deletion inttest/scaling-etcd/scaling_etcd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (s *ScalingSuite) TestK0sGetsUp() {
err = wait.PollImmediateUntilWithContext(s.Context(), 1*time.Second, func(ctx context.Context) (bool, error) {
etcdSts, err := kc.AppsV1().StatefulSets("default").Get(s.Context(), "kmc-scaling-etcd", metav1.GetOptions{})
if err != nil {
return false, err
return false, nil
}

return etcdSts.Status.Replicas == 3, nil
Expand Down

0 comments on commit 90e5907

Please sign in to comment.