Add envtest for testing controllers

Signed-off-by: Adrian Pedriza <[email protected]>

cmd/main.go

@@ -203,7 +203,6 @@ func main() {
 
 		if err = (&controlplane.K0sController{
 			Client:     mgr.GetClient(),
-			Scheme:     mgr.GetScheme(),
 			ClientSet:  clientSet,
 			RESTConfig: restConfig,
 		}).SetupWithManager(mgr); err != nil {

go.mod

@@ -5,18 +5,23 @@ go 1.22.0
 require (
 	github.com/cloudflare/cfssl v1.6.4
 	github.com/go-logr/logr v1.4.2
+	github.com/gobuffalo/flect v1.0.2
 	github.com/google/uuid v1.6.0
 	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/k0s v1.27.2-0.20230504131248-94378e521a29
 	github.com/k0sproject/rig v0.18.7
 	github.com/k0sproject/version v0.6.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.4
+	k8s.io/apiextensions-apiserver v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
+	k8s.io/klog/v2 v2.100.1
+	k8s.io/kubectl v0.28.4
 	k8s.io/kubernetes v1.28.4
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	sigs.k8s.io/controller-runtime v0.16.5
@@ -25,7 +30,7 @@ require (
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver v1.5.0
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
@@ -56,7 +61,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.24.0
 	gotest.tools/v3 v3.4.0 // indirect
 	helm.sh/helm/v3 v3.11.3 // indirect
 	k8s.io/kube-aggregator v0.27.2 // indirect
@@ -90,7 +95,6 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -124,7 +128,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
@@ -151,6 +154,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
@@ -167,14 +171,12 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/cloud-provider v0.27.1 // indirect
 	k8s.io/cluster-bootstrap v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/component-helpers v0.28.4 // indirect
 	k8s.io/controller-manager v0.28.4 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kms v0.28.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/kubelet v0.27.1 // indirect

go.sum

@@ -551,6 +551,8 @@ k8s.io/kube-aggregator v0.28.4 h1:VIGTKc3cDaJ44bvj988MTapJyRPbWXXcCvlp7HVLq5Q=
 k8s.io/kube-aggregator v0.28.4/go.mod h1:SHehggsYGjVaE1CZTfhukAPpdhs7bflJiddLrabbQNY=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/kubectl v0.28.4 h1:gWpUXW/T7aFne+rchYeHkyB8eVDl5UZce8G4X//kjUQ=
+k8s.io/kubectl v0.28.4/go.mod h1:CKOccVx3l+3MmDbkXtIUtibq93nN2hkDR99XDCn7c/c=
 k8s.io/kubelet v0.28.4 h1:Ypxy1jaFlSXFXbg/yVtFOU2ZxErBVRJfLu8+t4s7Dtw=
 k8s.io/kubelet v0.28.4/go.mod h1:w1wPI12liY/aeC70nqKYcNNkr6/nbyvdMB7P7wmww2o=
 k8s.io/kubernetes v1.28.4 h1:aRNxs5jb8FVTtlnxeA4FSDBVKuFwA8Gw40/U2zReBYA=

internal/controller/controlplane/helper.go

@@ -29,7 +29,7 @@ func (c *K0sController) createMachine(ctx context.Context, name string, cluster
 	if err != nil {
 		return nil, fmt.Errorf("error generating machine: %w", err)
 	}
-	_ = ctrl.SetControllerReference(kcp, machine, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, machine, c.Client.Scheme())
 
 	return machine, c.Client.Patch(ctx, machine, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",
@@ -171,7 +171,11 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		return nil, err
 	}
 
-	_ = ctrl.SetControllerReference(kcp, unstructuredMachineTemplate, c.Scheme)
+	_ = ctrl.SetControllerReference(cluster, unstructuredMachineTemplate, c.Client.Scheme())
+	err = c.Client.Patch(ctx, unstructuredMachineTemplate, client.Merge, &client.PatchOptions{FieldManager: "k0smotron"})
+	if err != nil {
+		return nil, err
+	}
 
 	template, found, err := unstructured.NestedMap(unstructuredMachineTemplate.UnstructuredContent(), "spec", "template")
 	if !found {
@@ -180,17 +184,17 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		return nil, fmt.Errorf("error getting spec.template map on %v %q: %w", unstructuredMachineTemplate.GroupVersionKind(), unstructuredMachineTemplate.GetName(), err)
 	}
 
-	machine := &unstructured.Unstructured{Object: template}
-	machine.SetName(name)
-	machine.SetNamespace(kcp.Namespace)
+	machineFromTemplate := &unstructured.Unstructured{Object: template}
+	machineFromTemplate.SetName(name)
+	machineFromTemplate.SetNamespace(kcp.Namespace)
 
 	annotations := map[string]string{}
 	for key, value := range kcp.Annotations {
 		annotations[key] = value
 	}
 	annotations[clusterv1.TemplateClonedFromNameAnnotation] = kcp.Spec.MachineTemplate.InfrastructureRef.Name
 	annotations[clusterv1.TemplateClonedFromGroupKindAnnotation] = kcp.Spec.MachineTemplate.InfrastructureRef.GroupVersionKind().GroupKind().String()
-	machine.SetAnnotations(annotations)
+	machineFromTemplate.SetAnnotations(annotations)
 
 	labels := map[string]string{}
 	for k, v := range kcp.Spec.MachineTemplate.ObjectMeta.Labels {
@@ -200,12 +204,12 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 	labels[clusterv1.ClusterNameLabel] = cluster.GetName()
 	labels[clusterv1.MachineControlPlaneLabel] = ""
 	labels[clusterv1.MachineControlPlaneNameLabel] = kcp.Name
-	machine.SetLabels(labels)
+	machineFromTemplate.SetLabels(labels)
 
-	machine.SetAPIVersion(unstructuredMachineTemplate.GetAPIVersion())
-	machine.SetKind(strings.TrimSuffix(unstructuredMachineTemplate.GetKind(), clusterv1.TemplateSuffix))
+	machineFromTemplate.SetAPIVersion(unstructuredMachineTemplate.GetAPIVersion())
+	machineFromTemplate.SetKind(strings.TrimSuffix(unstructuredMachineTemplate.GetKind(), clusterv1.TemplateSuffix))
 
-	return machine, nil
+	return machineFromTemplate, nil
 }
 
 func (c *K0sController) markChildControlNodeToLeave(ctx context.Context, name string, clientset *kubernetes.Clientset) error {

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -31,7 +31,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -57,13 +56,20 @@ const (
 	defaultK0sVersion = "v1.27.9+k0s.0"
 )
 
-var ErrNewMachinesNotReady = fmt.Errorf("waiting for new machines")
+var (
+	ErrNewMachinesNotReady    = fmt.Errorf("waiting for new machines")
+	FRPTokenNameTemplate      = "%s-frp-token"
+	FRPConfigMapNameTemplate  = "%s-frps-config"
+	FRPDeploymentNameTemplate = "%s-frps"
+	FRPServiceNameTemplate    = "%s-frps"
+)
 
 type K0sController struct {
 	client.Client
-	Scheme     *runtime.Scheme
 	ClientSet  *kubernetes.Clientset
 	RESTConfig *rest.Config
+	// workloadClusterKubeClient is used during testing to inject a fake client
+	workloadClusterKubeClient *kubernetes.Clientset
 }
 
 // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=k0scontrolplanes/status,verbs=get;list;watch;create;update;patch;delete
@@ -116,6 +122,11 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 		return ctrl.Result{}, nil
 	}
 
+	if annotations.IsPaused(cluster, kcp) {
+		log.Info("Reconciliation is paused for this object or owning cluster")
+		return ctrl.Result{}, nil
+	}
+
 	// Always patch the object to update the status
 	defer func() {
 		log.Info("Updating status")
@@ -147,11 +158,6 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 
 	log = log.WithValues("cluster", cluster.Name)
 
-	if annotations.IsPaused(cluster, kcp) {
-		log.Info("Reconciliation is paused for this object or owning cluster")
-		return ctrl.Result{}, nil
-	}
-
 	if err := c.ensureCertificates(ctx, cluster, kcp); err != nil {
 		log.Error(err, "Failed to ensure certificates")
 		return ctrl.Result{}, err
@@ -245,14 +251,14 @@ func (c *K0sController) reconcile(ctx context.Context, cluster *clusterv1.Cluste
 		return kcp.Status.Replicas, err
 	}
 
-	replicasToReport, err := c.reconcileMachines(ctx, cluster, kcp)
+	err = c.reconcileKubeconfig(ctx, cluster, kcp)
 	if err != nil {
-		return replicasToReport, err
+		return kcp.Spec.Replicas, fmt.Errorf("error reconciling kubeconfig secret: %w", err)
 	}
 
-	err = c.reconcileKubeconfig(ctx, cluster, kcp)
+	replicasToReport, err := c.reconcileMachines(ctx, cluster, kcp)
 	if err != nil {
-		return replicasToReport, fmt.Errorf("error reconciling kubeconfig secret: %w", err)
+		return replicasToReport, err
 	}
 
 	return replicasToReport, nil
@@ -605,7 +611,7 @@ token = ` + frpToken + `
 `
 	}
 
-	frpsCMName := kcp.GetName() + "-frps-config"
+	frpsCMName := fmt.Sprintf(FRPConfigMapNameTemplate, kcp.GetName())
 	cm := corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",
@@ -620,7 +626,7 @@ token = ` + frpToken + `
 		},
 	}
 
-	_ = ctrl.SetControllerReference(kcp, &cm, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &cm, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &cm, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating ConfigMap: %w", err)
@@ -632,7 +638,7 @@ token = ` + frpToken + `
 			Kind:       "Deployment",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPDeploymentNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: appsv1.DeploymentSpec{
@@ -689,7 +695,7 @@ token = ` + frpToken + `
 				}},
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsDeployment, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Deployment: %w", err)
@@ -701,7 +707,7 @@ token = ` + frpToken + `
 			Kind:       "Service",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPServiceNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: corev1.ServiceSpec{
@@ -725,7 +731,7 @@ token = ` + frpToken + `
 			Type: corev1.ServiceTypeNodePort,
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsService, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Service: %w", err)
@@ -744,7 +750,7 @@ func (c *K0sController) detectNodeIP(ctx context.Context, _ *cpv1beta1.K0sContro
 }
 
 func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (string, error) {
-	secretName := cluster.Name + "-frp-token"
+	secretName := fmt.Sprintf(FRPTokenNameTemplate, cluster.Name)
 
 	var existingSecret corev1.Secret
 	err := c.Client.Get(ctx, client.ObjectKey{Name: secretName, Namespace: cluster.Namespace}, &existingSecret)
@@ -773,7 +779,7 @@ func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.C
 		Type: clusterv1.ClusterSecretType,
 	}
 
-	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Client.Scheme())
 
 	return frpToken, c.Client.Patch(ctx, frpSecret, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",