From 9237196f531a6a71fb87f33e4b98520662b9a89d Mon Sep 17 00:00:00 2001 From: Adrian Pedriza Date: Fri, 20 Dec 2024 14:42:08 +0100 Subject: [PATCH] Report status of k0smotroncontrolplane replicas Signed-off-by: Adrian Pedriza --- api/controlplane/v1beta1/k0smotron_types.go | 28 ++++- .../v1beta1/k0smotroncluster_types.go | 4 +- ...uster.x-k8s.io_k0smotroncontrolplanes.yaml | 27 +++- ...uster.x-k8s.io_k0smotroncontrolplanes.yaml | 27 +++- docs/resource-reference.md | 44 ++++++- .../k0smotron_controlplane_controller.go | 119 +++++++++++++++++- .../k0smotroncluster_configmap.go | 5 +- .../k0smotroncluster_entrypoint.go | 5 +- .../k0smotron.io/k0smotroncluster_etcd.go | 17 +-- .../k0smotroncluster_kubeconfig.go | 5 +- .../k0smotroncluster_monitoring_config.go | 5 +- .../k0smotron.io/k0smotroncluster_service.go | 5 +- .../k0smotroncluster_statefulset.go | 9 +- internal/controller/k0smotron.io/util.go | 45 ------- internal/controller/util/util.go | 35 ++++++ ...capi_docker_clusterclass_k0smotron_test.go | 33 +++++ 16 files changed, 334 insertions(+), 79 deletions(-) delete mode 100644 internal/controller/k0smotron.io/util.go create mode 100644 internal/controller/util/util.go diff --git a/api/controlplane/v1beta1/k0smotron_types.go b/api/controlplane/v1beta1/k0smotron_types.go index 92f7833c9..3d800b56a 100644 --- a/api/controlplane/v1beta1/k0smotron_types.go +++ b/api/controlplane/v1beta1/k0smotron_types.go @@ -50,9 +50,27 @@ type K0smotronControlPlaneList struct { type K0smotronControlPlaneStatus struct { // Ready denotes that the control plane is ready - Ready bool `json:"ready"` - ControlPlaneReady bool `json:"controlPlaneReady"` - Inititalized bool `json:"initialized"` - ExternalManagedControlPlane bool `json:"externalManagedControlPlane"` - Version string `json:"version"` + Ready bool `json:"ready"` + ControlPlaneReady bool `json:"controlPlaneReady"` + Inititalized bool `json:"initialized"` + ExternalManagedControlPlane bool `json:"externalManagedControlPlane"` + // version represents the minimum Kubernetes version for the control plane pods + // in the cluster. + // +optional + Version string `json:"version"` + // replicas is the total number of pods targeted by this control plane + // +optional + Replicas int32 `json:"replicas"` + // updatedReplicas is the total number of pods targeted by this control plane + // that have the desired version. + // +optional + UpdatedReplicas int32 `json:"updatedReplicas"` + // readyReplicas is the total number of fully running and ready control plane pods. + // +optional + ReadyReplicas int32 `json:"readyReplicas"` + // unavailableReplicas is the total number of unavailable pods targeted by this control plane. + // This is the total number of pods with Condition Ready = false. + // They may either be pods that are running but not yet ready. + // +optional + UnavailableReplicas int32 `json:"unavailableReplicas"` } diff --git a/api/k0smotron.io/v1beta1/k0smotroncluster_types.go b/api/k0smotron.io/v1beta1/k0smotroncluster_types.go index dee636b96..dc31dfbea 100644 --- a/api/k0smotron.io/v1beta1/k0smotroncluster_types.go +++ b/api/k0smotron.io/v1beta1/k0smotroncluster_types.go @@ -116,7 +116,7 @@ type Mount struct { const ( defaultK0SImage = "k0sproject/k0s" defaultK0SVersion = "v1.27.9-k0s.0" - defaultK0SSuffix = "k0s.0" + DefaultK0SSuffix = "k0s.0" ) func (c *ClusterSpec) GetImage() string { @@ -126,7 +126,7 @@ func (c *ClusterSpec) GetImage() string { } if !strings.Contains(k0sVersion, "-k0s.") { - k0sVersion = fmt.Sprintf("%s-%s", k0sVersion, defaultK0SSuffix) + k0sVersion = fmt.Sprintf("%s-%s", k0sVersion, DefaultK0SSuffix) } if c.Image == "" { diff --git a/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml b/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml index 1908f051a..91388af4c 100644 --- a/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml +++ b/config/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml @@ -4314,14 +4314,39 @@ spec: ready: description: Ready denotes that the control plane is ready type: boolean + readyReplicas: + description: readyReplicas is the total number of fully running and + ready control plane pods. + format: int32 + type: integer + replicas: + description: replicas is the total number of pods targeted by this + control plane + format: int32 + type: integer + unavailableReplicas: + description: |- + unavailableReplicas is the total number of unavailable pods targeted by this control plane. + This is the total number of pods with Condition Ready = false. + They may either be pods that are running but not yet ready. + format: int32 + type: integer + updatedReplicas: + description: |- + updatedReplicas is the total number of pods targeted by this control plane + that have the desired version. + format: int32 + type: integer version: + description: |- + version represents the minimum Kubernetes version for the control plane pods + in the cluster. type: string required: - controlPlaneReady - externalManagedControlPlane - initialized - ready - - version type: object type: object served: true diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml index 1908f051a..91388af4c 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml @@ -4314,14 +4314,39 @@ spec: ready: description: Ready denotes that the control plane is ready type: boolean + readyReplicas: + description: readyReplicas is the total number of fully running and + ready control plane pods. + format: int32 + type: integer + replicas: + description: replicas is the total number of pods targeted by this + control plane + format: int32 + type: integer + unavailableReplicas: + description: |- + unavailableReplicas is the total number of unavailable pods targeted by this control plane. + This is the total number of pods with Condition Ready = false. + They may either be pods that are running but not yet ready. + format: int32 + type: integer + updatedReplicas: + description: |- + updatedReplicas is the total number of pods targeted by this control plane + that have the desired version. + format: int32 + type: integer version: + description: |- + version represents the minimum Kubernetes version for the control plane pods + in the cluster. type: string required: - controlPlaneReady - externalManagedControlPlane - initialized - ready - - version type: object type: object served: true diff --git a/docs/resource-reference.md b/docs/resource-reference.md index 8a0a56d21..d2edbce3c 100644 --- a/docs/resource-reference.md +++ b/docs/resource-reference.md @@ -11554,13 +11554,53 @@ will pick it automatically.
Ready denotes that the control plane is ready
true + + readyReplicas + integer + + readyReplicas is the total number of fully running and ready control plane pods.
+
+ Format: int32
+ + false + + replicas + integer + + replicas is the total number of pods targeted by this control plane
+
+ Format: int32
+ + false + + unavailableReplicas + integer + + unavailableReplicas is the total number of unavailable pods targeted by this control plane. +This is the total number of pods with Condition Ready = false. +They may either be pods that are running but not yet ready.
+
+ Format: int32
+ + false + + updatedReplicas + integer + + updatedReplicas is the total number of pods targeted by this control plane +that have the desired version.
+
+ Format: int32
+ + false version string -
+ version represents the minimum Kubernetes version for the control plane pods +in the cluster.
- true + false diff --git a/internal/controller/controlplane/k0smotron_controlplane_controller.go b/internal/controller/controlplane/k0smotron_controlplane_controller.go index ea4c16463..65db2bb9e 100644 --- a/internal/controller/controlplane/k0smotron_controlplane_controller.go +++ b/internal/controller/controlplane/k0smotron_controlplane_controller.go @@ -18,11 +18,14 @@ package controlplane import ( "context" + "errors" "fmt" "reflect" + "strings" "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -45,6 +48,8 @@ import ( cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1" kapi "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + "github.com/k0sproject/k0smotron/internal/exec" + "github.com/k0sproject/version" ) type K0smotronController struct { @@ -107,6 +112,26 @@ func (c *K0smotronController) Reconcile(ctx context.Context, req ctrl.Request) ( // return ctrl.Result{}, nil // } + defer func() { + derr := c.updateStatus(ctx, types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, kcp) + if derr != nil { + if errors.Is(derr, ErrNotReady) { + res = ctrl.Result{RequeueAfter: 10 * time.Second, Requeue: true} + } else { + log.Error(derr, "Failed to update K0smotronContorlPlane status") + err = derr + return + } + } + + derr = c.Status().Patch(ctx, kcp, client.Merge) + if derr != nil { + log.Error(derr, "Failed to patch K0smotronContorlPlane status") + err = derr + return + } + }() + res, ready, err := c.reconcile(ctx, cluster, kcp) if err != nil { log.Error(err, "Reconciliation failed") @@ -147,7 +172,6 @@ func (c *K0smotronController) Reconcile(ctx context.Context, req ctrl.Request) ( kcp.Status.ExternalManagedControlPlane = true kcp.Status.Inititalized = true kcp.Status.ControlPlaneReady = true - kcp.Status.Version = kcp.Spec.Version err = c.Status().Update(ctx, kcp) return res, err @@ -300,6 +324,99 @@ func (c *K0smotronController) ensureCertificates(ctx context.Context, cluster *c return certificates.LookupOrGenerate(ctx, c.Client, capiutil.ObjectKey(cluster), *metav1.NewControllerRef(kcp, cpv1beta1.GroupVersion.WithKind("K0smotronControlPlane"))) } +func (c *K0smotronController) updateStatus(ctx context.Context, cluster types.NamespacedName, kcp *cpv1beta1.K0smotronControlPlane) error { + var kCluster kapi.Cluster + err := c.Client.Get(ctx, cluster, &kCluster) + if err != nil { + if apierrors.IsNotFound(err) { + // The cluster is not yet created. + return nil + } + + return err + } + + contolPlanePods := &corev1.PodList{} + opts := &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + "cluster.x-k8s.io/cluster-name": cluster.Name, + "cluster.x-k8s.io/control-plane": "true", + }), + } + err = c.List(ctx, contolPlanePods, opts) + if err != nil { + return err + } + + kcp.Status.Replicas = int32(len(contolPlanePods.Items)) + + var ( + updatedReplicas, readyReplicas, unavailableReplicas int + ) + + desiredVersionStr := kcp.Spec.Version + if !strings.Contains(desiredVersionStr, "+") { + desiredVersionStr = fmt.Sprintf("%s+%s", desiredVersionStr, kapi.DefaultK0SSuffix) + } + desiredVersion, err := version.NewVersion(desiredVersionStr) + if err != nil { + return err + } + minimumVersion := *desiredVersion + + for _, pod := range contolPlanePods.Items { + isPodReady := false + for _, c := range pod.Status.Conditions { + // readiness probe in pod will propagate pod status Ready = True if k0s service is running successfully. + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + isPodReady = true + break + } + } + if isPodReady { + readyReplicas++ + } else { + unavailableReplicas++ + // if pod is unavailable subsequent checks do not apply + continue + } + + currentVersionOutput, err := exec.PodExecCmdOutput(ctx, c.ClientSet, c.RESTConfig, pod.GetName(), pod.GetNamespace(), "k0s version") + if err != nil { + return err + } + currentVersionStr, _ := strings.CutSuffix(currentVersionOutput, "\n") + currentVersion, err := version.NewVersion(currentVersionStr) + if err != nil { + return err + } + + if desiredVersion.Equal(currentVersion) { + updatedReplicas++ + } + + if currentVersion.LessThan(&minimumVersion) { + minimumVersion = *currentVersion + } + } + + kcp.Status.UpdatedReplicas = int32(updatedReplicas) + kcp.Status.ReadyReplicas = int32(readyReplicas) + kcp.Status.UnavailableReplicas = int32(unavailableReplicas) + + if kcp.Status.ReadyReplicas > 0 { + kcp.Status.Version = minimumVersion.String() + } + + // if no replicas are yet available or the desired version is not in the current state of the + // control plane, the reconciliation is requeued waiting for the desired replicas to become available. + if kcp.Status.UnavailableReplicas > 0 || desiredVersion.String() != kcp.Status.Version { + return ErrNotReady + } + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (c *K0smotronController) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controller/k0smotron.io/k0smotroncluster_configmap.go b/internal/controller/k0smotron.io/k0smotroncluster_configmap.go index 27bb5398e..c12b5e557 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_configmap.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_configmap.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/yaml" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" "github.com/k0sproject/k0smotron/internal/util" ) @@ -99,8 +100,8 @@ func (r *ClusterReconciler) generateConfig(kmc *km.Cluster, sans []string) (v1.C ObjectMeta: metav1.ObjectMeta{ Name: kmc.GetConfigMapName(), Namespace: kmc.Namespace, - Labels: labelsForCluster(kmc), - Annotations: annotationsForCluster(kmc), + Labels: kcontrollerutil.LabelsForK0smotronCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Data: map[string]string{ "K0SMOTRON_K0S_YAML": string(b), diff --git a/internal/controller/k0smotron.io/k0smotroncluster_entrypoint.go b/internal/controller/k0smotron.io/k0smotroncluster_entrypoint.go index f2b0df97a..4f42ec5a6 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_entrypoint.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_entrypoint.go @@ -23,6 +23,7 @@ import ( "text/template" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -55,8 +56,8 @@ func (r *ClusterReconciler) generateEntrypointCM(kmc *km.Cluster) (v1.ConfigMap, ObjectMeta: metav1.ObjectMeta{ Name: kmc.GetEntrypointConfigMapName(), Namespace: kmc.Namespace, - Labels: labelsForCluster(kmc), - Annotations: annotationsForCluster(kmc), + Labels: kcontrollerutil.LabelsForK0smotronCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Data: map[string]string{ "k0smotron-entrypoint.sh": entrypointBuf.String(), diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go index 198545f55..d029313f0 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go @@ -27,6 +27,7 @@ import ( km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -66,7 +67,7 @@ func (r *ClusterReconciler) reconcileEtcd(ctx context.Context, kmc *km.Cluster) } func (r *ClusterReconciler) reconcileEtcdSvc(ctx context.Context, kmc *km.Cluster) error { - labels := labelsForEtcdCluster(kmc) + labels := kcontrollerutil.LabelsForEtcdK0smotronCluster(kmc) svc := v1.Service{ TypeMeta: metav1.TypeMeta{ @@ -77,7 +78,7 @@ func (r *ClusterReconciler) reconcileEtcdSvc(ctx context.Context, kmc *km.Cluste Name: kmc.GetEtcdServiceName(), Namespace: kmc.Namespace, Labels: labels, - Annotations: annotationsForCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, @@ -105,7 +106,7 @@ func (r *ClusterReconciler) reconcileEtcdSvc(ctx context.Context, kmc *km.Cluste } func (r *ClusterReconciler) reconcileEtcdDefragJob(ctx context.Context, kmc *km.Cluster) error { - labels := labelsForEtcdCluster(kmc) + labels := kcontrollerutil.LabelsForEtcdK0smotronCluster(kmc) cronJob := batchv1.CronJob{ TypeMeta: metav1.TypeMeta{ @@ -116,7 +117,7 @@ func (r *ClusterReconciler) reconcileEtcdDefragJob(ctx context.Context, kmc *km. Name: kmc.GetEtcdDefragJobName(), Namespace: kmc.Namespace, Labels: labels, - Annotations: annotationsForCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Spec: batchv1.CronJobSpec{ Schedule: kmc.Spec.Etcd.DefragJob.Schedule, @@ -215,7 +216,7 @@ func (r *ClusterReconciler) reconcileEtcdStatefulSet(ctx context.Context, kmc *k } func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas int32) apps.StatefulSet { - labels := labelsForEtcdCluster(kmc) + labels := kcontrollerutil.LabelsForEtcdK0smotronCluster(kmc) size := kmc.Spec.Etcd.Persistence.Size @@ -255,7 +256,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in Name: kmc.GetEtcdStatefulSetName(), Namespace: kmc.Namespace, Labels: labels, - Annotations: annotationsForCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Spec: apps.StatefulSetSpec{ ServiceName: kmc.GetEtcdServiceName(), @@ -277,7 +278,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in PodAffinityTerm: v1.PodAffinityTerm{ TopologyKey: "topology.kubernetes.io/zone", LabelSelector: &metav1.LabelSelector{ - MatchLabels: labelsForEtcdCluster(kmc), + MatchLabels: kcontrollerutil.LabelsForEtcdK0smotronCluster(kmc), }, }, }, @@ -286,7 +287,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in PodAffinityTerm: v1.PodAffinityTerm{ TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{ - MatchLabels: labelsForEtcdCluster(kmc), + MatchLabels: kcontrollerutil.LabelsForEtcdK0smotronCluster(kmc), }, }, }, diff --git a/internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go b/internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go index 7b8bbb13c..a1ce64d32 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go @@ -20,6 +20,7 @@ import ( "context" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" "github.com/k0sproject/k0smotron/internal/exec" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,8 +54,8 @@ func (r *ClusterReconciler) reconcileKubeConfigSecret(ctx context.Context, kmc k ObjectMeta: metav1.ObjectMeta{ Name: kmc.GetAdminConfigSecretName(), Namespace: kmc.Namespace, - Labels: labelsForCluster(&kmc), - Annotations: annotationsForCluster(&kmc), + Labels: kcontrollerutil.LabelsForK0smotronCluster(&kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(&kmc), }, StringData: map[string]string{"value": output}, Type: clusterv1.ClusterSecretType, diff --git a/internal/controller/k0smotron.io/k0smotroncluster_monitoring_config.go b/internal/controller/k0smotron.io/k0smotroncluster_monitoring_config.go index 6308e6062..d90674c77 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_monitoring_config.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_monitoring_config.go @@ -22,6 +22,7 @@ import ( "text/template" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -56,8 +57,8 @@ func (r *ClusterReconciler) generateMonitoringCM(kmc *km.Cluster) (v1.ConfigMap, ObjectMeta: metav1.ObjectMeta{ Name: kmc.GetMonitoringConfigMapName(), Namespace: kmc.Namespace, - Labels: labelsForCluster(kmc), - Annotations: annotationsForCluster(kmc), + Labels: kcontrollerutil.LabelsForK0smotronCluster(kmc), + Annotations: kcontrollerutil.AnnotationsForK0smotronCluster(kmc), }, Data: map[string]string{ "prometheus.yml": entrypointBuf.String(), diff --git a/internal/controller/k0smotron.io/k0smotroncluster_service.go b/internal/controller/k0smotron.io/k0smotroncluster_service.go index ea37a4530..998606b71 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_service.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_service.go @@ -24,6 +24,7 @@ import ( "github.com/k0sproject/k0smotron/internal/controller/util" km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -89,7 +90,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service { // Copy both Cluster level labels and Service labels labels := map[string]string{} - for k, v := range labelsForCluster(kmc) { + for k, v := range kcontrollerutil.LabelsForK0smotronCluster(kmc) { labels[k] = v } for k, v := range kmc.Spec.Service.Labels { @@ -98,7 +99,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service { // Copy both Cluster level annotations and Service annotations annotations := map[string]string{} - for k, v := range annotationsForCluster(kmc) { + for k, v := range kcontrollerutil.AnnotationsForK0smotronCluster(kmc) { annotations[k] = v } for k, v := range kmc.Spec.Service.Annotations { diff --git a/internal/controller/k0smotron.io/k0smotroncluster_statefulset.go b/internal/controller/k0smotron.io/k0smotroncluster_statefulset.go index 2ff1e6d7a..b6a2448cb 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_statefulset.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_statefulset.go @@ -26,6 +26,7 @@ import ( "github.com/k0sproject/k0smotron/internal/controller/util" "k8s.io/utils/ptr" + kcontrollerutil "github.com/k0sproject/k0smotron/internal/controller/util" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -51,8 +52,8 @@ func (r *ClusterReconciler) findStatefulSetPod(ctx context.Context, statefulSet func (r *ClusterReconciler) generateStatefulSet(kmc *km.Cluster) (apps.StatefulSet, error) { - labels := labelsForCluster(kmc) - annotations := annotationsForCluster(kmc) + labels := kcontrollerutil.LabelsForK0smotronControlPlane(kmc) + annotations := kcontrollerutil.AnnotationsForK0smotronCluster(kmc) statefulSet := apps.StatefulSet{ TypeMeta: metav1.TypeMeta{ @@ -83,7 +84,7 @@ func (r *ClusterReconciler) generateStatefulSet(kmc *km.Cluster) (apps.StatefulS PodAffinityTerm: v1.PodAffinityTerm{ TopologyKey: "topology.kubernetes.io/zone", LabelSelector: &metav1.LabelSelector{ - MatchLabels: defaultClusterLabels(kmc), + MatchLabels: kcontrollerutil.DefaultK0smotronClusterLabels(kmc), }, }, }, @@ -92,7 +93,7 @@ func (r *ClusterReconciler) generateStatefulSet(kmc *km.Cluster) (apps.StatefulS PodAffinityTerm: v1.PodAffinityTerm{ TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{ - MatchLabels: defaultClusterLabels(kmc), + MatchLabels: kcontrollerutil.DefaultK0smotronClusterLabels(kmc), }, }, }, diff --git a/internal/controller/k0smotron.io/util.go b/internal/controller/k0smotron.io/util.go deleted file mode 100644 index 41ae1f487..000000000 --- a/internal/controller/k0smotron.io/util.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package k0smotronio - -import km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" - -func defaultClusterLabels(kmc *km.Cluster) map[string]string { - return map[string]string{ - "app": "k0smotron", - "cluster": kmc.Name, - } -} - -func labelsForCluster(kmc *km.Cluster) map[string]string { - labels := defaultClusterLabels(kmc) - for k, v := range kmc.Labels { - labels[k] = v - } - labels["component"] = "cluster" - return labels -} - -func labelsForEtcdCluster(kmc *km.Cluster) map[string]string { - labels := labelsForCluster(kmc) - labels["component"] = "etcd" - return labels -} - -func annotationsForCluster(kmc *km.Cluster) map[string]string { - return kmc.Annotations -} diff --git a/internal/controller/util/util.go b/internal/controller/util/util.go new file mode 100644 index 000000000..311232d9d --- /dev/null +++ b/internal/controller/util/util.go @@ -0,0 +1,35 @@ +package util + +import km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1" + +func DefaultK0smotronClusterLabels(kmc *km.Cluster) map[string]string { + return map[string]string{ + "app": "k0smotron", + "cluster": kmc.Name, + } +} + +func LabelsForK0smotronCluster(kmc *km.Cluster) map[string]string { + labels := DefaultK0smotronClusterLabels(kmc) + for k, v := range kmc.Labels { + labels[k] = v + } + labels["component"] = "cluster" + return labels +} + +func LabelsForK0smotronControlPlane(kmc *km.Cluster) map[string]string { + labels := LabelsForK0smotronCluster(kmc) + labels["cluster.x-k8s.io/control-plane"] = "true" + return labels +} + +func LabelsForEtcdK0smotronCluster(kmc *km.Cluster) map[string]string { + labels := LabelsForK0smotronCluster(kmc) + labels["component"] = "etcd" + return labels +} + +func AnnotationsForK0smotronCluster(kmc *km.Cluster) map[string]string { + return kmc.Annotations +} diff --git a/inttest/capi-docker-clusterclass-k0smotron/capi_docker_clusterclass_k0smotron_test.go b/inttest/capi-docker-clusterclass-k0smotron/capi_docker_clusterclass_k0smotron_test.go index c553ee33d..54304db27 100644 --- a/inttest/capi-docker-clusterclass-k0smotron/capi_docker_clusterclass_k0smotron_test.go +++ b/inttest/capi-docker-clusterclass-k0smotron/capi_docker_clusterclass_k0smotron_test.go @@ -21,10 +21,15 @@ import ( "os" "os/exec" "testing" + "time" + controlplanev1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1" "github.com/k0sproject/k0smotron/inttest/util" "github.com/stretchr/testify/suite" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -87,6 +92,32 @@ func (s *CAPIDockerClusterClassK0smotronSuite) TestCAPIDocker() { // Wait for the cluster to be ready // Wait to see the CP pods ready s.Require().NoError(util.WaitForStatefulSet(s.ctx, s.client, "kmc-docker-test-cluster", "default")) + + crdConfig := *s.restConfig + crdConfig.ContentConfig.GroupVersion = &controlplanev1beta1.GroupVersion + crdConfig.APIPath = "/apis" + crdConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme) + crdConfig.UserAgent = rest.DefaultKubernetesUserAgent() + crdRestClient, err := rest.UnversionedRESTClientFor(&crdConfig) + s.Require().NoError(err) + err = wait.PollUntilContextCancel(s.ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { + var kcps controlplanev1beta1.K0smotronControlPlaneList + err = crdRestClient.Get().Resource("k0smotroncontrolplanes").Namespace("default").Do(ctx).Into(&kcps) + if err != nil || len(kcps.Items) == 0 { + return false, nil + } + + kcp := kcps.Items[0] + + ready := kcp.Status.ReadyReplicas == 2 && + kcp.Status.UnavailableReplicas == 0 && + kcp.Status.Ready && + kcp.Status.UpdatedReplicas == 2 && + kcp.Status.Version == "v1.27.2+k0s.0" + + return ready, nil + }) + s.Require().NoError(err) } func (s *CAPIDockerClusterClassK0smotronSuite) applyClusterObjects() { @@ -116,6 +147,8 @@ spec: topology: class: k0smotron-cluster-class version: v1.27.2 + controlPlane: + replicas: 2 workers: machineDeployments: - class: docker-test-default-worker