Add envtest for testing controllers

Signed-off-by: Adrian Pedriza <[email protected]>
k0sproject · Nov 29, 2024 · 51d1e3c · 51d1e3c
1 parent aef0261
commit 51d1e3c
Show file tree

Hide file tree

Showing 11 changed files with 1,714 additions and 30 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -203,7 +203,6 @@ func main() {
 
 		if err = (&controlplane.K0sController{
 			Client:     mgr.GetClient(),
-			Scheme:     mgr.GetScheme(),
 			ClientSet:  clientSet,
 			RESTConfig: restConfig,
 		}).SetupWithManager(mgr); err != nil {

diff --git a/go.mod b/go.mod
@@ -5,18 +5,23 @@ go 1.22.0
 require (
 	github.com/cloudflare/cfssl v1.6.4
 	github.com/go-logr/logr v1.4.2
+	github.com/gobuffalo/flect v1.0.2
 	github.com/google/uuid v1.6.0
 	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/k0s v1.27.2-0.20230504131248-94378e521a29
 	github.com/k0sproject/rig v0.18.7
 	github.com/k0sproject/version v0.6.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.4
+	k8s.io/apiextensions-apiserver v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
+	k8s.io/klog/v2 v2.100.1
+	k8s.io/kubectl v0.28.4
 	k8s.io/kubernetes v1.28.4
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	sigs.k8s.io/controller-runtime v0.16.5
@@ -25,7 +30,7 @@ require (
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver v1.5.0
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
@@ -56,7 +61,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.24.0
 	gotest.tools/v3 v3.4.0 // indirect
 	helm.sh/helm/v3 v3.11.3 // indirect
 	k8s.io/kube-aggregator v0.27.2 // indirect
@@ -90,7 +95,6 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -124,7 +128,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
@@ -151,6 +154,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
@@ -167,14 +171,12 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/cloud-provider v0.27.1 // indirect
 	k8s.io/cluster-bootstrap v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/component-helpers v0.28.4 // indirect
 	k8s.io/controller-manager v0.28.4 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kms v0.28.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/kubelet v0.27.1 // indirect

diff --git a/go.sum b/go.sum
@@ -551,6 +551,8 @@ k8s.io/kube-aggregator v0.28.4 h1:VIGTKc3cDaJ44bvj988MTapJyRPbWXXcCvlp7HVLq5Q=
 k8s.io/kube-aggregator v0.28.4/go.mod h1:SHehggsYGjVaE1CZTfhukAPpdhs7bflJiddLrabbQNY=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/kubectl v0.28.4 h1:gWpUXW/T7aFne+rchYeHkyB8eVDl5UZce8G4X//kjUQ=
+k8s.io/kubectl v0.28.4/go.mod h1:CKOccVx3l+3MmDbkXtIUtibq93nN2hkDR99XDCn7c/c=
 k8s.io/kubelet v0.28.4 h1:Ypxy1jaFlSXFXbg/yVtFOU2ZxErBVRJfLu8+t4s7Dtw=
 k8s.io/kubelet v0.28.4/go.mod h1:w1wPI12liY/aeC70nqKYcNNkr6/nbyvdMB7P7wmww2o=
 k8s.io/kubernetes v1.28.4 h1:aRNxs5jb8FVTtlnxeA4FSDBVKuFwA8Gw40/U2zReBYA=

diff --git a/internal/controller/controlplane/helper.go b/internal/controller/controlplane/helper.go
@@ -43,14 +43,15 @@ import (
 
 const (
 	etcdMemberConditionTypeJoined = "Joined"
+	generatedMachineRoleLabel     = "cluster.x-k8s.io/generateMachine-role"
 )
 
 func (c *K0sController) createMachine(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane, infraRef corev1.ObjectReference, failureDomain *string) (*clusterv1.Machine, error) {
 	machine, err := c.generateMachine(ctx, name, cluster, kcp, infraRef, failureDomain)
 	if err != nil {
 		return nil, fmt.Errorf("error generating machine: %w", err)
 	}
-	_ = ctrl.SetControllerReference(kcp, machine, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, machine, c.Client.Scheme())
 
 	return machine, c.Client.Patch(ctx, machine, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",
@@ -81,9 +82,9 @@ func (c *K0sController) generateMachine(_ context.Context, name string, cluster
 	v := kcp.Spec.Version
 
 	labels := map[string]string{
-		"cluster.x-k8s.io/cluster-name":         kcp.Name,
-		"cluster.x-k8s.io/control-plane":        "true",
-		"cluster.x-k8s.io/generateMachine-role": "control-plane",
+		clusterv1.ClusterNameLabel:         kcp.Name,
+		clusterv1.MachineControlPlaneLabel: "true",
+		generatedMachineRoleLabel:          "control-plane",
 	}
 
 	for _, arg := range kcp.Spec.K0sConfigSpec.Args {
@@ -210,7 +211,11 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		return nil, err
 	}
 
-	_ = ctrl.SetControllerReference(kcp, unstructuredMachineTemplate, c.Scheme)
+	_ = ctrl.SetControllerReference(cluster, unstructuredMachineTemplate, c.Client.Scheme())
+	err = c.Client.Patch(ctx, unstructuredMachineTemplate, client.Merge, &client.PatchOptions{FieldManager: "k0smotron"})
+	if err != nil {
+		return nil, err
+	}
 
 	template, found, err := unstructured.NestedMap(unstructuredMachineTemplate.UnstructuredContent(), "spec", "template")
 	if !found {

diff --git a/internal/controller/controlplane/k0s_controlplane_controller.go b/internal/controller/controlplane/k0s_controlplane_controller.go
@@ -34,7 +34,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -63,15 +62,20 @@ const (
 )
 
 var (
-	ErrNotReady            = fmt.Errorf("waiting for the state")
-	ErrNewMachinesNotReady = fmt.Errorf("waiting for new machines: %w", ErrNotReady)
+	ErrNotReady               = fmt.Errorf("waiting for the state")
+	ErrNewMachinesNotReady    = fmt.Errorf("waiting for new machines: %w", ErrNotReady)
+	FRPTokenNameTemplate      = "%s-frp-token"
+	FRPConfigMapNameTemplate  = "%s-frps-config"
+	FRPDeploymentNameTemplate = "%s-frps"
+	FRPServiceNameTemplate    = "%s-frps"
 )
 
 type K0sController struct {
 	client.Client
-	Scheme     *runtime.Scheme
 	ClientSet  *kubernetes.Clientset
 	RESTConfig *rest.Config
+	// workloadClusterKubeClient is used during testing to inject a fake client
+	workloadClusterKubeClient *kubernetes.Clientset
 }
 
 // +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=k0scontrolplanes/status,verbs=get;list;watch;create;update;patch;delete
@@ -124,6 +128,11 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 		return ctrl.Result{}, nil
 	}
 
+	if annotations.IsPaused(cluster, kcp) {
+		log.Info("Reconciliation is paused for this object or owning cluster")
+		return ctrl.Result{}, nil
+	}
+
 	// Always patch the object to update the status
 	defer func() {
 		log.Info("Updating status")
@@ -160,11 +169,6 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 
 	log = log.WithValues("cluster", cluster.Name)
 
-	if annotations.IsPaused(cluster, kcp) {
-		log.Info("Reconciliation is paused for this object or owning cluster")
-		return ctrl.Result{}, nil
-	}
-
 	if err := c.ensureCertificates(ctx, cluster, kcp); err != nil {
 		log.Error(err, "Failed to ensure certificates")
 		return ctrl.Result{}, err
@@ -630,7 +634,7 @@ token = ` + frpToken + `
 `
 	}
 
-	frpsCMName := kcp.GetName() + "-frps-config"
+	frpsCMName := fmt.Sprintf(FRPConfigMapNameTemplate, kcp.GetName())
 	cm := corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",
@@ -645,7 +649,7 @@ token = ` + frpToken + `
 		},
 	}
 
-	_ = ctrl.SetControllerReference(kcp, &cm, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &cm, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &cm, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating ConfigMap: %w", err)
@@ -657,7 +661,7 @@ token = ` + frpToken + `
 			Kind:       "Deployment",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPDeploymentNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: appsv1.DeploymentSpec{
@@ -714,7 +718,7 @@ token = ` + frpToken + `
 				}},
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsDeployment, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Deployment: %w", err)
@@ -726,7 +730,7 @@ token = ` + frpToken + `
 			Kind:       "Service",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      kcp.GetName() + "-frps",
+			Name:      fmt.Sprintf(FRPServiceNameTemplate, kcp.GetName()),
 			Namespace: kcp.GetNamespace(),
 		},
 		Spec: corev1.ServiceSpec{
@@ -750,7 +754,7 @@ token = ` + frpToken + `
 			Type: corev1.ServiceTypeNodePort,
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsService, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Service: %w", err)
@@ -769,7 +773,7 @@ func (c *K0sController) detectNodeIP(ctx context.Context, _ *cpv1beta1.K0sContro
 }
 
 func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (string, error) {
-	secretName := cluster.Name + "-frp-token"
+	secretName := fmt.Sprintf(FRPTokenNameTemplate, cluster.Name)
 
 	var existingSecret corev1.Secret
 	err := c.Client.Get(ctx, client.ObjectKey{Name: secretName, Namespace: cluster.Namespace}, &existingSecret)
@@ -798,7 +802,7 @@ func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.C
 		Type: clusterv1.ClusterSecretType,
 	}
 
-	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Client.Scheme())
 
 	return frpToken, c.Client.Patch(ctx, frpSecret, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",