From c2e0ebf520bc47e9ebb95ef37b7c249e80f4595e Mon Sep 17 00:00:00 2001 From: Alexey Makhov Date: Fri, 5 Jul 2024 15:14:12 +0300 Subject: [PATCH] Set labels to worker nodes to be able to identify machine name and set providerID to child cluster's nodes Signed-off-by: Alexey Makhov --- cmd/main.go | 7 ++ .../bootstrap/bootstrap_controller.go | 22 +++-- .../bootstrap/bootstrap_controller_test.go | 34 ++++--- .../controlplane_bootstrap_controller.go | 71 +++++++++----- .../bootstrap/providerid_controller.go | 94 +++++++++++++++++++ internal/controller/controlplane/helper.go | 20 +++- internal/controller/controlplane/util.go | 24 ++--- .../remote_machine_controller.go | 2 +- .../k0smotroncluster_controller.go | 8 +- internal/controller/util/kubeclient.go | 30 ++++++ inttest/Makefile | 3 +- .../capi_remote_machine_template_test.go | 12 ++- .../capi_remote_machine_test.go | 13 +++ 13 files changed, 272 insertions(+), 68 deletions(-) create mode 100644 internal/controller/bootstrap/providerid_controller.go create mode 100644 internal/controller/util/kubeclient.go diff --git a/cmd/main.go b/cmd/main.go index 88a9f3e75..2cfaccf25 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -181,6 +181,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Bootstrap") os.Exit(1) } + if err = (&bootstrap.ProviderIDController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Bootstrap") + os.Exit(1) + } } if isControllerEnabled(controlPlaneController) { diff --git a/internal/controller/bootstrap/bootstrap_controller.go b/internal/controller/bootstrap/bootstrap_controller.go index ee0a88745..e1ed57842 100644 --- a/internal/controller/bootstrap/bootstrap_controller.go +++ b/internal/controller/bootstrap/bootstrap_controller.go @@ -47,6 +47,8 @@ import ( const ( defaultK0sSuffix = "k0s.0" + + machineNameNodeLabel = "k0smotron.io/machine-name" ) type Controller struct { @@ -62,6 +64,13 @@ type Scope struct { Cluster *clusterv1.Cluster } +type ControllerScope struct { + Config *bootstrapv1.K0sControllerConfig + ConfigOwner *bsutil.ConfigOwner + Cluster *clusterv1.Cluster + WorkerEnabled bool +} + // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k0sworkerconfigs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k0sworkerconfigs/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch @@ -160,7 +169,7 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl. files = append(files, config.Spec.Files...) downloadCommands := createDownloadCommands(config) - installCmd := createInstallCmd(config) + installCmd := createInstallCmd(scope) commands := config.Spec.PreStartCommands commands = append(commands, downloadCommands...) @@ -289,7 +298,6 @@ func (r *Controller) getK0sToken(ctx context.Context, scope *Scope) (string, err ca := certificates.GetByPurpose(secret.ClusterCA) if ca.KeyPair == nil { return "", errors.New("failed to get CA certificate key pair") - } joinToken, err := kutil.CreateK0sJoinToken(ca.KeyPair.Cert, token, fmt.Sprintf("https://%s:%d", scope.Cluster.Spec.ControlPlaneEndpoint.Host, scope.Cluster.Spec.ControlPlaneEndpoint.Port), "kubelet-bootstrap") @@ -299,11 +307,13 @@ func (r *Controller) getK0sToken(ctx context.Context, scope *Scope) (string, err return joinToken, nil } -func createInstallCmd(config *bootstrapv1.K0sWorkerConfig) string { +func createInstallCmd(scope *Scope) string { installCmd := []string{ - "k0s install worker --token-file /etc/k0s.token"} - if config.Spec.Args != nil && len(config.Spec.Args) > 0 { - installCmd = append(installCmd, config.Spec.Args...) + "k0s install worker --token-file /etc/k0s.token", + "--labels=" + fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()), + } + if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 { + installCmd = append(installCmd, scope.Config.Spec.Args...) } return strings.Join(installCmd, " ") } diff --git a/internal/controller/bootstrap/bootstrap_controller_test.go b/internal/controller/bootstrap/bootstrap_controller_test.go index d6edae163..b5a3bb5ef 100644 --- a/internal/controller/bootstrap/bootstrap_controller_test.go +++ b/internal/controller/bootstrap/bootstrap_controller_test.go @@ -21,33 +21,45 @@ import ( bootstrapv1 "github.com/k0sproject/k0smotron/api/bootstrap/v1beta1" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + bsutil "sigs.k8s.io/cluster-api/bootstrap/util" ) func Test_createInstallCmd(t *testing.T) { - base := "k0s install worker --token-file /etc/k0s.token" + base := "k0s install worker --token-file /etc/k0s.token --labels=k0smotron.io/machine-name=test" tests := []struct { - name string - config *bootstrapv1.K0sWorkerConfig - want string + name string + scope *Scope + want string }{ { - name: "with default config", - config: &bootstrapv1.K0sWorkerConfig{}, - want: base, + name: "with default config", + scope: &Scope{ + Config: &bootstrapv1.K0sWorkerConfig{}, + ConfigOwner: &bsutil.ConfigOwner{Unstructured: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{"name": "test"}, + }}}, + }, + want: base, }, { name: "with args", - config: &bootstrapv1.K0sWorkerConfig{ - Spec: bootstrapv1.K0sWorkerConfigSpec{ - Args: []string{"--debug", "--labels=k0sproject.io/foo=bar"}, + scope: &Scope{ + Config: &bootstrapv1.K0sWorkerConfig{ + Spec: bootstrapv1.K0sWorkerConfigSpec{ + Args: []string{"--debug", "--labels=k0sproject.io/foo=bar"}, + }, }, + ConfigOwner: &bsutil.ConfigOwner{Unstructured: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{"name": "test"}, + }}}, }, want: base + " --debug --labels=k0sproject.io/foo=bar", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.want, createInstallCmd(tt.config)) + require.Equal(t, tt.want, createInstallCmd(tt.scope)) }) } } diff --git a/internal/controller/bootstrap/controlplane_bootstrap_controller.go b/internal/controller/bootstrap/controlplane_bootstrap_controller.go index 09c12816e..e076166a6 100644 --- a/internal/controller/bootstrap/controlplane_bootstrap_controller.go +++ b/internal/controller/bootstrap/controlplane_bootstrap_controller.go @@ -127,9 +127,18 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - scope := &Scope{ - ConfigOwner: configOwner, - Cluster: cluster, + scope := &ControllerScope{ + Config: config, + ConfigOwner: configOwner, + Cluster: cluster, + WorkerEnabled: false, + } + + for _, arg := range config.Spec.Args { + if arg == "--enable-worker" || arg == "--enable-worker=true" { + scope.WorkerEnabled = true + break + } } // TODO Check if the secret is already present etc. to bail out early @@ -190,16 +199,16 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request if err != nil { return ctrl.Result{}, fmt.Errorf("error generating initial control plane files: %v", err) } - installCmd = createCPInstallCmd(config) + installCmd = createCPInstallCmd(scope) } else { - files, err = c.genControlPlaneJoinFiles(ctx, scope, config, files) + files, err = c.genControlPlaneJoinFiles(ctx, scope, files) if err != nil { return ctrl.Result{}, fmt.Errorf("error generating control plane join files: %v", err) } - installCmd = createCPInstallCmdWithJoinToken(config, joinTokenFilePath) + installCmd = createCPInstallCmdWithJoinToken(scope, joinTokenFilePath) } if config.Spec.Tunneling.Enabled { - tunnelingFiles, err := c.genTunnelingFiles(ctx, scope, config) + tunnelingFiles, err := c.genTunnelingFiles(ctx, scope) if err != nil { return ctrl.Result{}, fmt.Errorf("error generating tunneling files: %v", err) } @@ -282,7 +291,7 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } -func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context, scope *Scope, files []cloudinit.File) ([]cloudinit.File, error) { +func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File) ([]cloudinit.File, error) { log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name) certs, _, err := c.getCerts(ctx, scope) @@ -295,7 +304,7 @@ func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context return files, nil } -func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *Scope, config *bootstrapv1.K0sControllerConfig, files []cloudinit.File) ([]cloudinit.File, error) { +func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File) ([]cloudinit.File, error) { log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name) _, ca, err := c.getCerts(ctx, scope) @@ -322,7 +331,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s return nil, err } - host, err := c.findFirstControllerIP(ctx, config) + host, err := c.findFirstControllerIP(ctx, scope.Config) if err != nil { log.Error(err, "Failed to get controller IP") return nil, err @@ -341,7 +350,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s return files, err } -func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *Scope, kcs *bootstrapv1.K0sControllerConfig) ([]cloudinit.File, error) { +func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *ControllerScope) ([]cloudinit.File, error) { secretName := scope.Cluster.Name + "-frp-token" frpSecret := corev1.Secret{} err := c.Client.Get(ctx, client.ObjectKey{Namespace: scope.Cluster.Namespace, Name: secretName}, &frpSecret) @@ -351,7 +360,7 @@ func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *S frpToken := string(frpSecret.Data["value"]) var modeConfig string - if kcs.Spec.Tunneling.Mode == "proxy" { + if scope.Config.Spec.Tunneling.Mode == "proxy" { modeConfig = fmt.Sprintf(` type = tcpmux custom_domains = %s @@ -419,11 +428,11 @@ spec: return []cloudinit.File{{ Path: "/var/lib/k0s/manifests/k0smotron-tunneling/manifest.yaml", Permissions: "0644", - Content: fmt.Sprintf(tunnelingResources, kcs.Spec.Tunneling.ServerAddress, kcs.Spec.Tunneling.ServerNodePort, frpToken, modeConfig), + Content: fmt.Sprintf(tunnelingResources, scope.Config.Spec.Tunneling.ServerAddress, scope.Config.Spec.Tunneling.ServerNodePort, frpToken, modeConfig), }}, nil } -func (c *ControlPlaneController) getCerts(ctx context.Context, scope *Scope) ([]cloudinit.File, *secret.Certificate, error) { +func (c *ControlPlaneController) getCerts(ctx context.Context, scope *ControllerScope) ([]cloudinit.File, *secret.Certificate, error) { var files []cloudinit.File certificates := secret.NewCertificatesForInitialControlPlane(&kubeadmbootstrapv1.ClusterConfiguration{ CertificatesDir: "/var/lib/k0s/pki", @@ -505,31 +514,45 @@ func createCPDownloadCommands(config *bootstrapv1.K0sControllerConfig) []string return []string{"curl -sSfL https://get.k0s.sh | sh"} } -func createCPInstallCmd(config *bootstrapv1.K0sControllerConfig) string { +func createCPInstallCmd(scope *ControllerScope) string { installCmd := []string{ "k0s install controller", "--force", "--enable-dynamic-config", - "--env AUTOPILOT_HOSTNAME=" + config.Name, - "--kubelet-extra-args=--hostname-override=" + config.Name, + "--env AUTOPILOT_HOSTNAME=" + scope.Config.Name, + } + + if scope.WorkerEnabled { + installCmd = append(installCmd, + "--kubelet-extra-args=--hostname-override="+scope.Config.Name, + "--labels="+fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()), + ) } - if config.Spec.Args != nil && len(config.Spec.Args) > 0 { - installCmd = append(installCmd, config.Spec.Args...) + + if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 { + installCmd = append(installCmd, scope.Config.Spec.Args...) } return strings.Join(installCmd, " ") } -func createCPInstallCmdWithJoinToken(config *bootstrapv1.K0sControllerConfig, tokenPath string) string { +func createCPInstallCmdWithJoinToken(scope *ControllerScope, tokenPath string) string { installCmd := []string{ "k0s install controller", "--force", "--enable-dynamic-config", - "--env AUTOPILOT_HOSTNAME=" + config.Name, - "--kubelet-extra-args=--hostname-override=" + config.Name, + "--env AUTOPILOT_HOSTNAME=" + scope.Config.Name, } + + if scope.WorkerEnabled { + installCmd = append(installCmd, + "--kubelet-extra-args=--hostname-override="+scope.Config.Name, + "--labels="+fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()), + ) + } + installCmd = append(installCmd, "--token-file", tokenPath) - if config.Spec.Args != nil && len(config.Spec.Args) > 0 { - installCmd = append(installCmd, config.Spec.Args...) + if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 { + installCmd = append(installCmd, scope.Config.Spec.Args...) } return strings.Join(installCmd, " ") } diff --git a/internal/controller/bootstrap/providerid_controller.go b/internal/controller/bootstrap/providerid_controller.go new file mode 100644 index 000000000..896de8096 --- /dev/null +++ b/internal/controller/bootstrap/providerid_controller.go @@ -0,0 +1,94 @@ +package bootstrap + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/retry" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + capiutil "sigs.k8s.io/cluster-api/util" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + k0smoutil "github.com/k0sproject/k0smotron/internal/controller/util" +) + +type ProviderIDController struct { + client.Client + Scheme *runtime.Scheme +} + +func (p *ProviderIDController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx).WithValues("providerID", req.NamespacedName) + log.Info("Reconciling machine's ProviderID") + + machine := &clusterv1.Machine{} + if err := p.Get(ctx, req.NamespacedName, machine); err != nil { + if apierrors.IsNotFound(err) { + log.Info("machine not found") + return ctrl.Result{}, nil + } + log.Error(err, "Failed to get machine") + return ctrl.Result{}, err + } + + // Skip the control plane machines that don't have worker enabled + if capiutil.IsControlPlaneMachine(machine) && machine.ObjectMeta.Labels["k0smotron.io/control-plane-worker-enabled"] != "true" { + return ctrl.Result{}, nil + } + + // Skip non-k0s machines + if machine.Spec.Bootstrap.ConfigRef.Kind != "K0sControllerConfig" && machine.Spec.Bootstrap.ConfigRef.Kind != "K0sWorkerConfig" { + return ctrl.Result{}, nil + } + + if machine.Spec.ProviderID == nil || *machine.Spec.ProviderID == "" { + return ctrl.Result{}, fmt.Errorf("waiting for providerID for the machine %s/%s", machine.Namespace, machine.Name) + } + + cluster, err := capiutil.GetClusterByName(ctx, p.Client, machine.Namespace, machine.Spec.ClusterName) + if err != nil { + return ctrl.Result{}, fmt.Errorf("can't get cluster %s/%s: %w", machine.Namespace, machine.Spec.ClusterName, err) + } + + childClient, err := k0smoutil.GetKubeClient(context.Background(), p.Client, cluster) + if err != nil { + return ctrl.Result{}, fmt.Errorf("can't get kube client for cluster %s/%s: %w. may not be created yet", machine.Namespace, machine.Spec.ClusterName, err) + } + + nodes, err := childClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", machineNameNodeLabel, machine.GetName()), + }) + if err != nil || len(nodes.Items) == 0 { + log.Info("waiting for node to be available for machine " + machine.Name) + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil + } + + node := nodes.Items[0] + if node.Spec.ProviderID == "" { + node.Spec.ProviderID = *machine.Spec.ProviderID + err = retry.OnError(retry.DefaultBackoff, func(err error) bool { + return true + }, func() error { + _, upErr := childClient.CoreV1().Nodes().Update(context.Background(), &node, metav1.UpdateOptions{}) + return upErr + }) + + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update node '%s' with providerID: %w", node.Name, err) + } + } + + return ctrl.Result{}, nil +} + +func (p *ProviderIDController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&clusterv1.Machine{}). + Complete(p) +} diff --git a/internal/controller/controlplane/helper.go b/internal/controller/controlplane/helper.go index dbf962c2b..d0042df19 100644 --- a/internal/controller/controlplane/helper.go +++ b/internal/controller/controlplane/helper.go @@ -72,6 +72,20 @@ func (c *K0sController) generateMachine(_ context.Context, name string, cluster return nil, fmt.Errorf("error parsing version %q: %w", kcp.Spec.Version, err) } v := fmt.Sprintf("%d.%d.%d", ver.Major(), ver.Minor(), ver.Patch()) + + labels := map[string]string{ + "cluster.x-k8s.io/cluster-name": kcp.Name, + "cluster.x-k8s.io/control-plane": "true", + "cluster.x-k8s.io/generateMachine-role": "control-plane", + } + + for _, arg := range kcp.Spec.K0sConfigSpec.Args { + if arg == "--enable-worker" || arg == "--enable-worker=true" { + labels["k0smotron.io/control-plane-worker-enabled"] = "true" + break + } + } + return &clusterv1.Machine{ TypeMeta: metav1.TypeMeta{ APIVersion: clusterv1.GroupVersion.String(), @@ -80,11 +94,7 @@ func (c *K0sController) generateMachine(_ context.Context, name string, cluster ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: kcp.Namespace, - Labels: map[string]string{ - "cluster.x-k8s.io/cluster-name": kcp.Name, - "cluster.x-k8s.io/control-plane": "true", - "cluster.x-k8s.io/generateMachine-role": "control-plane", - }, + Labels: labels, }, Spec: clusterv1.MachineSpec{ Version: &v, diff --git a/internal/controller/controlplane/util.go b/internal/controller/controlplane/util.go index 942162707..2b4bb9dc6 100644 --- a/internal/controller/controlplane/util.go +++ b/internal/controller/controlplane/util.go @@ -3,21 +3,22 @@ package controlplane import ( "context" "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/util" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/certs" "sigs.k8s.io/cluster-api/util/kubeconfig" "sigs.k8s.io/cluster-api/util/secret" "sigs.k8s.io/controller-runtime/pkg/client" cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1" + k0smoutil "github.com/k0sproject/k0smotron/internal/controller/util" ) func (c *K0sController) getMachineTemplate(ctx context.Context, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) { @@ -91,18 +92,5 @@ func (c *K0sController) createKubeconfigSecret(ctx context.Context, cfg *api.Con } func (c *K0sController) getKubeClient(ctx context.Context, cluster *clusterv1.Cluster) (*kubernetes.Clientset, error) { - data, err := kubeconfig.FromSecret(ctx, c.Client, client.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Name}) - if err != nil { - return nil, fmt.Errorf("error fetching %s kubeconfig from secret: %w", cluster.Name, err) - } - config, err := clientcmd.NewClientConfigFromBytes(data) - if err != nil { - return nil, fmt.Errorf("error generating %s clientconfig: %w", cluster.Name, err) - } - restConfig, err := config.ClientConfig() - if err != nil { - return nil, fmt.Errorf("error generating %s restconfig: %w", cluster.Name, err) - } - - return kubernetes.NewForConfig(restConfig) + return k0smoutil.GetKubeClient(ctx, c.Client, cluster) } diff --git a/internal/controller/infrastructure/remote_machine_controller.go b/internal/controller/infrastructure/remote_machine_controller.go index e2ee4ea2d..8d9ddc99c 100644 --- a/internal/controller/infrastructure/remote_machine_controller.go +++ b/internal/controller/infrastructure/remote_machine_controller.go @@ -351,7 +351,7 @@ func (r *RemoteMachineController) returnMachineToPool(ctx context.Context, rm *i } } log := log.FromContext(ctx).WithValues("remotemachine", rm.Name) - log.Error(fmt.Errorf("no pooled machine found for remote machine"), rm.Namespace, rm.Name) + log.Error(fmt.Errorf("no pooled machine found for remote machine"), "pooled machine not found", "namespace", rm.Namespace, "name", rm.Name) return nil } diff --git a/internal/controller/k0smotron.io/k0smotroncluster_controller.go b/internal/controller/k0smotron.io/k0smotroncluster_controller.go index f2788920b..537b73831 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_controller.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_controller.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/secret" @@ -171,7 +172,12 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct func (r *ClusterReconciler) updateStatus(ctx context.Context, kmc km.Cluster, status string) { logger := log.FromContext(ctx) kmc.Status.ReconciliationStatus = status - if err := r.Status().Patch(ctx, &kmc, client.Merge); err != nil { + err := retry.OnError(retry.DefaultBackoff, func(err error) bool { + return true + }, func() error { + return r.Status().Patch(ctx, &kmc, client.Merge) + }) + if err != nil { logger.Error(err, fmt.Sprintf("Unable to update status: %s", status)) } } diff --git a/internal/controller/util/kubeclient.go b/internal/controller/util/kubeclient.go new file mode 100644 index 000000000..aeb4933d7 --- /dev/null +++ b/internal/controller/util/kubeclient.go @@ -0,0 +1,30 @@ +package util + +import ( + "context" + "fmt" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + capiutil "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/kubeconfig" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetKubeClient(ctx context.Context, client client.Client, cluster *clusterv1.Cluster) (*kubernetes.Clientset, error) { + data, err := kubeconfig.FromSecret(ctx, client, capiutil.ObjectKey(cluster)) + if err != nil { + return nil, fmt.Errorf("error fetching %s kubeconfig from secret: %w", cluster.Name, err) + } + config, err := clientcmd.NewClientConfigFromBytes(data) + if err != nil { + return nil, fmt.Errorf("error generating %s clientconfig: %w", cluster.Name, err) + } + restConfig, err := config.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error generating %s restconfig: %w", cluster.Name, err) + } + + return kubernetes.NewForConfig(restConfig) +} diff --git a/inttest/Makefile b/inttest/Makefile index f9a3d47df..995fdb72f 100644 --- a/inttest/Makefile +++ b/inttest/Makefile @@ -39,7 +39,8 @@ check-scaling-etcd: TIMEOUT=10m check-monitoring: TIMEOUT=7m check-ha-controlplane: TIMEOUT=7m check-capi-controlplane-docker-tunneling: TIMEOUT=10m -check-capi-remote-machine: TIMEOUT=8m +check-capi-remote-machine: TIMEOUT=12m +check-capi-remote-machine-template: TIMEOUT=12m check-capi-remote-machine-template-update: TIMEOUT=10m check-capi-docker-machine-template-update: TIMEOUT=10m check-capi-remote-machine-job-provision: TIMEOUT=10m diff --git a/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go b/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go index 030352bf6..4313af94c 100644 --- a/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go +++ b/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go @@ -152,6 +152,7 @@ func (s *RemoteMachineTemplateSuite) TestCAPIRemoteMachine() { // Verify the RemoteMachine is at expected state + expectedProviderID := fmt.Sprintf("remote-machine://%s:22", s.getWorkerIP()) // nolint:staticcheck err = wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (bool, error) { rm, err := s.getRemoteMachine("remote-test-0", "default") @@ -159,7 +160,6 @@ func (s *RemoteMachineTemplateSuite) TestCAPIRemoteMachine() { return false, err } - expectedProviderID := fmt.Sprintf("remote-machine://%s:22", s.getWorkerIP()) return rm.Status.Ready && expectedProviderID == rm.Spec.ProviderID, nil }) s.Require().NoError(err) @@ -167,6 +167,16 @@ func (s *RemoteMachineTemplateSuite) TestCAPIRemoteMachine() { s.T().Log("waiting for node to be ready") s.Require().NoError(common.WaitForNodeReadyStatus(ctx, kmcKC, "remote-test-0", corev1.ConditionTrue)) + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) { + node, err := kmcKC.CoreV1().Nodes().Get(ctx, "remote-test-0", metav1.GetOptions{}) + if err != nil { + return false, err + } + + return node.Labels["k0smotron.io/machine-name"] == "remote-test-0" && node.Spec.ProviderID == expectedProviderID, nil + }) + s.Require().NoError(err) + s.T().Log("deleting node from cluster") s.Require().NoError(s.deleteRemoteMachine("remote-test-0", "default")) diff --git a/inttest/capi-remote-machine/capi_remote_machine_test.go b/inttest/capi-remote-machine/capi_remote_machine_test.go index 78119b699..05f2b14fc 100644 --- a/inttest/capi-remote-machine/capi_remote_machine_test.go +++ b/inttest/capi-remote-machine/capi_remote_machine_test.go @@ -18,6 +18,7 @@ package capiremotemachine import ( "bytes" + "context" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -27,6 +28,7 @@ import ( "os" "testing" "text/template" + "time" "github.com/k0sproject/k0s/inttest/common" infra "github.com/k0sproject/k0smotron/api/infrastructure/v1beta1" @@ -37,6 +39,7 @@ import ( "golang.org/x/crypto/ssh" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -158,6 +161,16 @@ func (s *RemoteMachineSuite) TestCAPIRemoteMachine() { expectedProviderID := fmt.Sprintf("remote-machine://%s:22", s.getWorkerIP()) s.Require().Equal(expectedProviderID, rm.Spec.ProviderID) + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (done bool, err error) { + node, err := kmcKC.CoreV1().Nodes().Get(ctx, "k0smotron0", metav1.GetOptions{}) + if err != nil { + return false, err + } + + return node.Labels["k0smotron.io/machine-name"] == "remote-test-0" && node.Spec.ProviderID == expectedProviderID, nil + }) + s.Require().NoError(err) + s.T().Log("deleting node from cluster") s.Require().NoError(s.deleteRemoteMachine("remote-test-0", "default"))