PVC resizing support. WIP

Signed-off-by: Alexey Makhov <[email protected]>

.github/workflows/go.yml

@@ -97,6 +97,7 @@ jobs:
           - check-basic
           - check-config-update-hcp
           - check-hostpath
+          - check-pvc
           - check-ha-controller
           - check-ha-controller-etcd
           - check-ha-controller-secret

internal/controller/k0smotron.io/k0smotroncluster_controller.go

@@ -149,6 +149,11 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		})
 	}
 
+	if err := r.reconcilePVC(ctx, kmc); err != nil {
+		r.updateStatus(ctx, kmc, "Failed reconciling PVCs")
+		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
+	}
+
 	logger.Info("Reconciling etcd")
 	if err := r.reconcileEtcd(ctx, &kmc); err != nil {
 		r.updateStatus(ctx, kmc, fmt.Sprintf("Failed reconciling etcd, %+v", err))

internal/controller/k0smotron.io/k0smotroncluster_pvc.go

@@ -0,0 +1,171 @@
+package k0smotronio
+
+import (
+	"context"
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
+)
+
+func (r *ClusterReconciler) reconcilePVC(ctx context.Context, kmc km.Cluster) error {
+	logger := log.FromContext(ctx)
+	logger.Info("Reconciling PVC")
+
+	// volumeClaimTemplates are immutable, so we need to
+	// update PVC, then delete the StatefulSet with --cascade=orphan and recreate it
+
+	err := r.reconcileControlPlanePVC(ctx, kmc)
+	if err != nil {
+		return fmt.Errorf("failed to reconcile control plane PVC: %w", err)
+	}
+	err = r.reconcileEtcdPVC(ctx, kmc)
+	if err != nil {
+		return fmt.Errorf("failed to reconcile control plane PVC: %w", err)
+	}
+
+	return nil
+}
+
+func (r *ClusterReconciler) reconcileControlPlanePVC(ctx context.Context, kmc km.Cluster) error {
+	// Do nothing if the persistence type is not PVC
+	if kmc.Spec.Persistence.Type != "pvc" {
+		return nil
+	}
+
+	var sts appsv1.StatefulSet
+	err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetStatefulSetName()}, &sts)
+	if err != nil {
+		// Do nothing if StatefulSet does not exist yet
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+
+		return fmt.Errorf("failed to get statefulset: %w", err)
+	}
+
+	// Do nothing if the sizes match
+	if kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests.Storage().Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
+		return nil
+	}
+
+	// Update the PVC size
+	var allowExpansion *bool
+	for i := 0; i < int(kmc.Spec.Replicas); i++ {
+
+		if kmc.Spec.Persistence.PersistentVolumeClaim.Name == "" {
+			kmc.Spec.Persistence.PersistentVolumeClaim.Name = kmc.GetVolumeName()
+		}
+		name := fmt.Sprintf("%s-%s-%d", kmc.Spec.Persistence.PersistentVolumeClaim.Name, kmc.GetStatefulSetName(), i)
+
+		var pvc corev1.PersistentVolumeClaim
+		err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
+		if err != nil {
+			return fmt.Errorf("failed to get PVC: %w", err)
+		}
+
+		if allowExpansion == nil {
+			var sc storagev1.StorageClass
+			err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
+			if err != nil {
+				return fmt.Errorf("failed to get StorageClass: %w", err)
+			}
+			allowExpansion = sc.AllowVolumeExpansion
+		}
+
+		if *allowExpansion {
+			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Persistence.PersistentVolumeClaim.Spec.Resources.Requests[corev1.ResourceStorage]
+			err = r.Update(ctx, &pvc)
+			if err != nil {
+				return fmt.Errorf("failed to update PVC: %w", err)
+			}
+
+			// Remove pod to trigger file system resize
+			err = r.Delete(ctx, &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
+					Namespace: kmc.Namespace,
+				},
+			}, &client.DeleteOptions{})
+
+			if err != nil {
+				return fmt.Errorf("failed to delete pod for resizing: %w", err)
+			}
+		} else {
+			break
+		}
+	}
+
+	return r.Delete(ctx, &sts, &client.DeleteOptions{PropagationPolicy: ptr.To(metav1.DeletePropagationOrphan)})
+}
+
+func (r *ClusterReconciler) reconcileEtcdPVC(ctx context.Context, kmc km.Cluster) error {
+	var sts appsv1.StatefulSet
+	err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: kmc.GetEtcdStatefulSetName()}, &sts)
+	if err != nil {
+		// Do nothing if StatefulSet does not exist yet
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+
+		return fmt.Errorf("failed to get etcd statefulset: %w", err)
+	}
+
+	// Do nothing if the sizes match
+	if kmc.Spec.Etcd.Persistence.Size.Cmp(*sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage()) == 0 {
+		return nil
+	}
+
+	// Update the PVC size
+	var allowExpansion *bool
+	for i := 0; i < int(calculateDesiredReplicas(&kmc)); i++ {
+		var pvc corev1.PersistentVolumeClaim
+
+		name := fmt.Sprintf("etcd-data-%s-%d", kmc.GetStatefulSetName(), i)
+		err := r.Get(ctx, client.ObjectKey{Namespace: kmc.Namespace, Name: name}, &pvc)
+		if err != nil {
+			return fmt.Errorf("failed to get etcd PVC: %w", err)
+		}
+
+		if allowExpansion == nil {
+			var sc storagev1.StorageClass
+			err = r.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, &sc)
+			if err != nil {
+				return fmt.Errorf("failed to get etcd StorageClass: %w", err)
+			}
+			allowExpansion = sc.AllowVolumeExpansion
+		}
+
+		if *allowExpansion {
+			pvc.Spec.Resources.Requests[corev1.ResourceStorage] = kmc.Spec.Etcd.Persistence.Size
+			err = r.Update(ctx, &pvc)
+			if err != nil {
+				return fmt.Errorf("failed to update etcd PVC: %w", err)
+			}
+
+			// Remove pod to trigger file system resize
+			err = r.Delete(ctx, &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-%d", kmc.GetStatefulSetName(), i),
+					Namespace: kmc.Namespace,
+				},
+			}, &client.DeleteOptions{})
+
+			if err != nil {
+				return fmt.Errorf("failed to delete etcd pod for resizing: %w", err)
+			}
+		} else {
+			break
+		}
+	}
+
+	return r.Delete(ctx, &sts, &client.DeleteOptions{PropagationPolicy: ptr.To(metav1.DeletePropagationOrphan)})
+}

inttest/Makefile.variables

@@ -8,6 +8,7 @@ smoketests := \
     check-basic \
     check-config-update-hcp \
     check-hostpath \
+    check-pvc \
     check-ha-controller \
     check-ha-controller-etcd \
     check-ha-controller-secret \

inttest/pvc/pvc_test.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvc
+
+import (
+	"context"
+	"testing"
+
+	"github.com/k0sproject/k0s/inttest/common"
+	"github.com/stretchr/testify/suite"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
+	"github.com/k0sproject/k0smotron/inttest/util"
+)
+
+type PVCSuite struct {
+	common.FootlooseSuite
+}
+
+func (s *PVCSuite) TestK0sGetsUp() {
+	s.T().Log("starting k0s")
+	s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server"))
+	s.Require().NoError(s.RunWorkers())
+
+	kc, err := s.KubeClient(s.ControllerNode(0))
+	s.Require().NoError(err)
+	rc, err := s.GetKubeConfig(s.ControllerNode(0))
+	s.Require().NoError(err)
+
+	err = s.WaitForNodeReady(s.WorkerNode(0), kc)
+	s.NoError(err)
+
+	// create folder for k0smotron persistent volume
+	ssh, err := s.SSH(s.Context(), s.WorkerNode(0))
+	s.Require().NoError(err)
+	defer ssh.Disconnect()
+	_, err = ssh.ExecWithOutput(s.Context(), "mkdir -p /tmp/kmc-test")
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.ImportK0smotronImages(s.Context()))
+
+	s.T().Log("deploying k0smotron operator")
+	s.Require().NoError(util.InstallK0smotronOperator(s.Context(), kc, rc))
+	s.Require().NoError(common.WaitForDeployment(s.Context(), kc, "k0smotron-controller-manager", "k0smotron"))
+
+	s.T().Log("deploying k0smotron cluster")
+	s.createK0smotronCluster(s.Context(), kc)
+	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
+
+	s.T().Log("updating k0smotron cluster")
+	s.updateK0smotronCluster(s.Context(), rc)
+
+	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
+	sts, err := kc.AppsV1().StatefulSets("kmc-test").Get(s.Context(), "kmc-kmc-test", metav1.GetOptions{})
+	s.Require().NoError(err)
+
+	s.Require().Equal("200Mi", sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().String())
+}
+
+func TestPVCSuite(t *testing.T) {
+	s := PVCSuite{
+		common.FootlooseSuite{
+			ControllerCount:                 1,
+			WorkerCount:                     1,
+			K0smotronWorkerCount:            1,
+			K0smotronImageBundleMountPoints: []string{"/dist/bundle.tar"},
+		},
+	}
+	suite.Run(t, &s)
+}
+
+func (s *PVCSuite) createK0smotronCluster(ctx context.Context, kc *kubernetes.Clientset) {
+	// create K0smotron namespace
+	_, err := kc.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "kmc-test",
+		},
+	}, metav1.CreateOptions{})
+	s.Require().NoError(err)
+
+	kmc := []byte(`
+	{
+		"apiVersion": "k0smotron.io/v1beta1",
+		"kind": "Cluster",
+		"metadata": {
+		  "name": "kmc-test",
+		  "namespace": "kmc-test"
+		},
+		"spec": {
+			"etcd":{
+				"persistence": {"size": "200Mi"}
+			},
+			"persistence": {
+				"type": "pvc",
+				"persistentVolumeClaim": {
+					"spec": {
+						"accessModes": ["ReadWriteOnce"],
+						"resources": {
+							"requests": {
+								"storage": "50Mi"
+							}
+						}
+					}
+				}
+			},
+			"k0sConfig": {
+				"apiVersion": "k0s.k0sproject.io/v1beta1",
+				"kind": "ClusterConfig",
+				"spec": {
+					"telemetry": {"enabled": false}
+				}
+			}
+		}
+	  }
+`)
+
+	res := kc.RESTClient().Post().AbsPath("/apis/k0smotron.io/v1beta1/namespaces/kmc-test/clusters").Body(kmc).Do(ctx)
+	s.Require().NoError(res.Error())
+}
+
+func (s *PVCSuite) updateK0smotronCluster(ctx context.Context, rc *rest.Config) {
+	crdConfig := *rc
+	crdConfig.ContentConfig.GroupVersion = &km.GroupVersion
+	crdConfig.APIPath = "/apis"
+	crdConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme)
+	crdConfig.UserAgent = rest.DefaultKubernetesUserAgent()
+	crdRestClient, err := rest.UnversionedRESTClientFor(&crdConfig)
+	s.Require().NoError(err)
+
+	patch := `[{"op": "replace", "path": "/spec/persistence/persistentVolumeClaim/spec/resources/requests/storage", "value": "200Mi"}]`
+	res := crdRestClient.
+		Patch(types.JSONPatchType).
+		Resource("clusters").
+		Name("kmc-test").
+		Namespace("kmc-test").
+		Body([]byte(patch)).
+		Do(ctx)
+	s.Require().NoError(res.Error())
+
+	patch = `[{"op": "replace", "path": "/spec/etcd/persistence/size", "value": "300Mi"}]`
+	res = crdRestClient.
+		Patch(types.JSONPatchType).
+		Resource("clusters").
+		Name("kmc-test").
+		Namespace("kmc-test").
+		Body([]byte(patch)).
+		Do(ctx)
+	s.Require().NoError(res.Error())
+}