Skip to content

Commit

Permalink
Set labels to worker nodes to be able to identify machine name and se…
Browse files Browse the repository at this point in the history
…t providerID to child cluster's nodes

Signed-off-by: Alexey Makhov <[email protected]>
  • Loading branch information
makhov committed Jul 12, 2024
1 parent f3e4abe commit 6196ed8
Show file tree
Hide file tree
Showing 13 changed files with 272 additions and 68 deletions.
7 changes: 7 additions & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,13 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "Bootstrap")
os.Exit(1)
}
if err = (&bootstrap.ProviderIDController{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Bootstrap")
os.Exit(1)
}
}

if isControllerEnabled(controlPlaneController) {
Expand Down
22 changes: 16 additions & 6 deletions internal/controller/bootstrap/bootstrap_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ import (

const (
defaultK0sSuffix = "k0s.0"

machineNameNodeLabel = "k0smotron.io/machine-name"
)

type Controller struct {
Expand All @@ -62,6 +64,13 @@ type Scope struct {
Cluster *clusterv1.Cluster
}

type ControllerScope struct {
Config *bootstrapv1.K0sControllerConfig
ConfigOwner *bsutil.ConfigOwner
Cluster *clusterv1.Cluster
WorkerEnabled bool
}

// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k0sworkerconfigs,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=k0sworkerconfigs/status,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch
Expand Down Expand Up @@ -160,7 +169,7 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.

files = append(files, config.Spec.Files...)
downloadCommands := createDownloadCommands(config)
installCmd := createInstallCmd(config)
installCmd := createInstallCmd(scope)

commands := config.Spec.PreStartCommands
commands = append(commands, downloadCommands...)
Expand Down Expand Up @@ -289,7 +298,6 @@ func (r *Controller) getK0sToken(ctx context.Context, scope *Scope) (string, err
ca := certificates.GetByPurpose(secret.ClusterCA)
if ca.KeyPair == nil {
return "", errors.New("failed to get CA certificate key pair")

}

joinToken, err := kutil.CreateK0sJoinToken(ca.KeyPair.Cert, token, fmt.Sprintf("https://%s:%d", scope.Cluster.Spec.ControlPlaneEndpoint.Host, scope.Cluster.Spec.ControlPlaneEndpoint.Port), "kubelet-bootstrap")
Expand All @@ -299,11 +307,13 @@ func (r *Controller) getK0sToken(ctx context.Context, scope *Scope) (string, err
return joinToken, nil
}

func createInstallCmd(config *bootstrapv1.K0sWorkerConfig) string {
func createInstallCmd(scope *Scope) string {
installCmd := []string{
"k0s install worker --token-file /etc/k0s.token"}
if config.Spec.Args != nil && len(config.Spec.Args) > 0 {
installCmd = append(installCmd, config.Spec.Args...)
"k0s install worker --token-file /etc/k0s.token",
"--labels=" + fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()),
}
if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 {
installCmd = append(installCmd, scope.Config.Spec.Args...)
}
return strings.Join(installCmd, " ")
}
Expand Down
34 changes: 23 additions & 11 deletions internal/controller/bootstrap/bootstrap_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,33 +21,45 @@ import (

bootstrapv1 "github.com/k0sproject/k0smotron/api/bootstrap/v1beta1"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
)

func Test_createInstallCmd(t *testing.T) {
base := "k0s install worker --token-file /etc/k0s.token"
base := "k0s install worker --token-file /etc/k0s.token --labels=k0smotron.io/machine-name=test"
tests := []struct {
name string
config *bootstrapv1.K0sWorkerConfig
want string
name string
scope *Scope
want string
}{
{
name: "with default config",
config: &bootstrapv1.K0sWorkerConfig{},
want: base,
name: "with default config",
scope: &Scope{
Config: &bootstrapv1.K0sWorkerConfig{},
ConfigOwner: &bsutil.ConfigOwner{Unstructured: &unstructured.Unstructured{Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": "test"},
}}},
},
want: base,
},
{
name: "with args",
config: &bootstrapv1.K0sWorkerConfig{
Spec: bootstrapv1.K0sWorkerConfigSpec{
Args: []string{"--debug", "--labels=k0sproject.io/foo=bar"},
scope: &Scope{
Config: &bootstrapv1.K0sWorkerConfig{
Spec: bootstrapv1.K0sWorkerConfigSpec{
Args: []string{"--debug", "--labels=k0sproject.io/foo=bar"},
},
},
ConfigOwner: &bsutil.ConfigOwner{Unstructured: &unstructured.Unstructured{Object: map[string]interface{}{
"metadata": map[string]interface{}{"name": "test"},
}}},
},
want: base + " --debug --labels=k0sproject.io/foo=bar",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.want, createInstallCmd(tt.config))
require.Equal(t, tt.want, createInstallCmd(tt.scope))
})
}
}
Expand Down
71 changes: 47 additions & 24 deletions internal/controller/bootstrap/controlplane_bootstrap_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,18 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

scope := &Scope{
ConfigOwner: configOwner,
Cluster: cluster,
scope := &ControllerScope{
Config: config,
ConfigOwner: configOwner,
Cluster: cluster,
WorkerEnabled: false,
}

for _, arg := range config.Spec.Args {
if arg == "--enable-worker" || arg == "--enable-worker=true" {
scope.WorkerEnabled = true
break
}
}

// TODO Check if the secret is already present etc. to bail out early
Expand Down Expand Up @@ -209,16 +218,16 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
if err != nil {
return ctrl.Result{}, fmt.Errorf("error generating initial control plane files: %v", err)
}
installCmd = createCPInstallCmd(config)
installCmd = createCPInstallCmd(scope)
} else {
files, err = c.genControlPlaneJoinFiles(ctx, scope, config, files)
files, err = c.genControlPlaneJoinFiles(ctx, scope, files)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error generating control plane join files: %v", err)
}
installCmd = createCPInstallCmdWithJoinToken(config, joinTokenFilePath)
installCmd = createCPInstallCmdWithJoinToken(scope, joinTokenFilePath)
}
if config.Spec.Tunneling.Enabled {
tunnelingFiles, err := c.genTunnelingFiles(ctx, scope, config)
tunnelingFiles, err := c.genTunnelingFiles(ctx, scope)
if err != nil {
return ctrl.Result{}, fmt.Errorf("error generating tunneling files: %v", err)
}
Expand Down Expand Up @@ -301,7 +310,7 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context, scope *Scope, files []cloudinit.File) ([]cloudinit.File, error) {
func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File) ([]cloudinit.File, error) {
log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name)

certs, _, err := c.getCerts(ctx, scope)
Expand All @@ -314,7 +323,7 @@ func (c *ControlPlaneController) genInitialControlPlaneFiles(ctx context.Context
return files, nil
}

func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *Scope, config *bootstrapv1.K0sControllerConfig, files []cloudinit.File) ([]cloudinit.File, error) {
func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, scope *ControllerScope, files []cloudinit.File) ([]cloudinit.File, error) {
log := log.FromContext(ctx).WithValues("K0sControllerConfig cluster", scope.Cluster.Name)

_, ca, err := c.getCerts(ctx, scope)
Expand All @@ -341,7 +350,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
return nil, err
}

host, err := c.findFirstControllerIP(ctx, config)
host, err := c.findFirstControllerIP(ctx, scope.Config)
if err != nil {
log.Error(err, "Failed to get controller IP")
return nil, err
Expand All @@ -360,7 +369,7 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
return files, err
}

func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *Scope, kcs *bootstrapv1.K0sControllerConfig) ([]cloudinit.File, error) {
func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *ControllerScope) ([]cloudinit.File, error) {
secretName := scope.Cluster.Name + "-frp-token"
frpSecret := corev1.Secret{}
err := c.Client.Get(ctx, client.ObjectKey{Namespace: scope.Cluster.Namespace, Name: secretName}, &frpSecret)
Expand All @@ -370,7 +379,7 @@ func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *S
frpToken := string(frpSecret.Data["value"])

var modeConfig string
if kcs.Spec.Tunneling.Mode == "proxy" {
if scope.Config.Spec.Tunneling.Mode == "proxy" {
modeConfig = fmt.Sprintf(`
type = tcpmux
custom_domains = %s
Expand Down Expand Up @@ -438,11 +447,11 @@ spec:
return []cloudinit.File{{
Path: "/var/lib/k0s/manifests/k0smotron-tunneling/manifest.yaml",
Permissions: "0644",
Content: fmt.Sprintf(tunnelingResources, kcs.Spec.Tunneling.ServerAddress, kcs.Spec.Tunneling.ServerNodePort, frpToken, modeConfig),
Content: fmt.Sprintf(tunnelingResources, scope.Config.Spec.Tunneling.ServerAddress, scope.Config.Spec.Tunneling.ServerNodePort, frpToken, modeConfig),
}}, nil
}

func (c *ControlPlaneController) getCerts(ctx context.Context, scope *Scope) ([]cloudinit.File, *secret.Certificate, error) {
func (c *ControlPlaneController) getCerts(ctx context.Context, scope *ControllerScope) ([]cloudinit.File, *secret.Certificate, error) {
var files []cloudinit.File
certificates := secret.NewCertificatesForInitialControlPlane(&kubeadmbootstrapv1.ClusterConfiguration{
CertificatesDir: "/var/lib/k0s/pki",
Expand Down Expand Up @@ -524,31 +533,45 @@ func createCPDownloadCommands(config *bootstrapv1.K0sControllerConfig) []string
return []string{"curl -sSfL https://get.k0s.sh | sh"}
}

func createCPInstallCmd(config *bootstrapv1.K0sControllerConfig) string {
func createCPInstallCmd(scope *ControllerScope) string {
installCmd := []string{
"k0s install controller",
"--force",
"--enable-dynamic-config",
"--env AUTOPILOT_HOSTNAME=" + config.Name,
"--kubelet-extra-args=--hostname-override=" + config.Name,
"--env AUTOPILOT_HOSTNAME=" + scope.Config.Name,
}

if scope.WorkerEnabled {
installCmd = append(installCmd,
"--kubelet-extra-args=--hostname-override="+scope.Config.Name,
"--labels="+fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()),
)
}
if config.Spec.Args != nil && len(config.Spec.Args) > 0 {
installCmd = append(installCmd, config.Spec.Args...)

if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 {
installCmd = append(installCmd, scope.Config.Spec.Args...)
}
return strings.Join(installCmd, " ")
}

func createCPInstallCmdWithJoinToken(config *bootstrapv1.K0sControllerConfig, tokenPath string) string {
func createCPInstallCmdWithJoinToken(scope *ControllerScope, tokenPath string) string {
installCmd := []string{
"k0s install controller",
"--force",
"--enable-dynamic-config",
"--env AUTOPILOT_HOSTNAME=" + config.Name,
"--kubelet-extra-args=--hostname-override=" + config.Name,
"--env AUTOPILOT_HOSTNAME=" + scope.Config.Name,
}

if scope.WorkerEnabled {
installCmd = append(installCmd,
"--kubelet-extra-args=--hostname-override="+scope.Config.Name,
"--labels="+fmt.Sprintf("%s=%s", machineNameNodeLabel, scope.ConfigOwner.GetName()),
)
}

installCmd = append(installCmd, "--token-file", tokenPath)
if config.Spec.Args != nil && len(config.Spec.Args) > 0 {
installCmd = append(installCmd, config.Spec.Args...)
if scope.Config.Spec.Args != nil && len(scope.Config.Spec.Args) > 0 {
installCmd = append(installCmd, scope.Config.Spec.Args...)
}
return strings.Join(installCmd, " ")
}
Expand Down
94 changes: 94 additions & 0 deletions internal/controller/bootstrap/providerid_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package bootstrap

import (
"context"
"fmt"
"time"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/util/retry"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capiutil "sigs.k8s.io/cluster-api/util"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"

k0smoutil "github.com/k0sproject/k0smotron/internal/controller/util"
)

type ProviderIDController struct {
client.Client
Scheme *runtime.Scheme
}

func (p *ProviderIDController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx).WithValues("providerID", req.NamespacedName)
log.Info("Reconciling machine's ProviderID")

machine := &clusterv1.Machine{}
if err := p.Get(ctx, req.NamespacedName, machine); err != nil {
if apierrors.IsNotFound(err) {
log.Info("machine not found")
return ctrl.Result{}, nil
}
log.Error(err, "Failed to get machine")
return ctrl.Result{}, err
}

// Skip the control plane machines that don't have worker enabled
if capiutil.IsControlPlaneMachine(machine) && machine.ObjectMeta.Labels["k0smotron.io/control-plane-worker-enabled"] != "true" {
return ctrl.Result{}, nil
}

// Skip non-k0s machines
if machine.Spec.Bootstrap.ConfigRef.Kind != "K0sControllerConfig" && machine.Spec.Bootstrap.ConfigRef.Kind != "K0sWorkerConfig" {
return ctrl.Result{}, nil
}

if machine.Spec.ProviderID == nil || *machine.Spec.ProviderID == "" {
return ctrl.Result{}, fmt.Errorf("waiting for providerID for the machine %s/%s", machine.Namespace, machine.Name)
}

cluster, err := capiutil.GetClusterByName(ctx, p.Client, machine.Namespace, machine.Spec.ClusterName)
if err != nil {
return ctrl.Result{}, fmt.Errorf("can't get cluster %s/%s: %w", machine.Namespace, machine.Spec.ClusterName, err)
}

childClient, err := k0smoutil.GetKubeClient(context.Background(), p.Client, cluster)
if err != nil {
return ctrl.Result{}, fmt.Errorf("can't get kube client for cluster %s/%s: %w. may not be created yet", machine.Namespace, machine.Spec.ClusterName, err)
}

nodes, err := childClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", machineNameNodeLabel, machine.GetName()),
})
if err != nil || len(nodes.Items) == 0 {
log.Info("waiting for node to be available for machine " + machine.Name)
return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 10}, nil
}

node := nodes.Items[0]
if node.Spec.ProviderID == "" {
node.Spec.ProviderID = *machine.Spec.ProviderID
err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
return true
}, func() error {
_, upErr := childClient.CoreV1().Nodes().Update(context.Background(), &node, metav1.UpdateOptions{})
return upErr
})

if err != nil {
return ctrl.Result{}, fmt.Errorf("failed to update node '%s' with providerID: %w", node.Name, err)
}
}

return ctrl.Result{}, nil
}

func (p *ProviderIDController) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&clusterv1.Machine{}).
Complete(p)
}
Loading

0 comments on commit 6196ed8

Please sign in to comment.