Client connection tunneling prototype. Proxy mode

Signed-off-by: Alexey Makhov <[email protected]>

.github/workflows/go.yml

@@ -107,6 +107,7 @@ jobs:
           - check-capi-controlplane-docker
           - check-capi-controlplane-docker-downscaling
           - check-capi-controlplane-docker-tunneling
+          - check-capi-controlplane-docker-tunneling-proxy
           - check-capi-controlplane-docker-worker
     steps:
       - name: Check out code into the Go module directory

api/bootstrap/v1beta1/k0s_types.go

@@ -198,4 +198,9 @@ type TunnelingSpec struct {
 	//+kubebuilder:validation:Optional
 	//+kubebuilder:default=31443
 	TunnelingNodePort int32 `json:"tunnelingNodePort,omitempty"`
+	// Mode describes tunneling mode.
+	// If empty, k0smotron will use the default one.
+	//+kubebuilder:validation:Enum=tunnel;proxy
+	//+kubebuilder:default=tunnel
+	Mode string `json:"mode,omitempty"`
 }

config/crd/bases/bootstrap.cluster.x-k8s.io_k0scontrollerconfigs.yaml

@@ -88,6 +88,14 @@ spec:
                     default: false
                     description: Enabled specifies whether tunneling is enabled.
                     type: boolean
+                  mode:
+                    default: tunnel
+                    description: Mode describes tunneling mode. If empty, k0smotron
+                      will use the default one.
+                    enum:
+                    - tunnel
+                    - proxy
+                    type: string
                   serverAddress:
                     description: Server address of the tunneling server. If empty,
                       k0smotron will try to detect worker node address for.

config/crd/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml

@@ -90,6 +90,14 @@ spec:
                         default: false
                         description: Enabled specifies whether tunneling is enabled.
                         type: boolean
+                      mode:
+                        default: tunnel
+                        description: Mode describes tunneling mode. If empty, k0smotron
+                          will use the default one.
+                        enum:
+                        - tunnel
+                        - proxy
+                        type: string
                       serverAddress:
                         description: Server address of the tunneling server. If empty,
                           k0smotron will try to detect worker node address for.

internal/controller/bootstrap/controlplane_bootstrap_controller.go

@@ -328,6 +328,19 @@ func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *S
 	}
 	frpToken := string(frpSecret.Data["value"])
 
+	var modeConfig string
+	if kcs.Spec.Tunneling.Mode == "proxy" {
+		modeConfig = fmt.Sprintf(`
+    type = tcpmux
+    custom_domains = %s
+    multiplexer = httpconnect
+`, scope.Cluster.Spec.ControlPlaneEndpoint.Host)
+	} else {
+		modeConfig = `
+    remote_port = 6443
+`
+	}
+
 	tunnelingResources := `
 ---
 apiVersion: v1
@@ -347,7 +360,7 @@ data:
     type = tcp
     local_ip = 10.96.0.1
     local_port = 443
-    remote_port = 6443
+    %s
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -384,7 +397,7 @@ spec:
 	return []cloudinit.File{{
 		Path:        "/var/lib/k0s/manifests/k0smotron-tunneling/manifest.yaml",
 		Permissions: "0644",
-		Content:     fmt.Sprintf(tunnelingResources, kcs.Spec.Tunneling.ServerAddress, kcs.Spec.Tunneling.ServerNodePort, frpToken),
+		Content:     fmt.Sprintf(tunnelingResources, kcs.Spec.Tunneling.ServerAddress, kcs.Spec.Tunneling.ServerNodePort, frpToken, modeConfig),
 	}}, nil
 }
 

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -128,7 +128,7 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 
 }
 
-func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster) error {
+func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) error {
 	if cluster.Spec.ControlPlaneEndpoint.IsZero() {
 		return errors.New("control plane endpoint is not set")
 	}
@@ -142,11 +142,53 @@ func (c *K0sController) reconcileKubeconfig(ctx context.Context, cluster *cluste
 		return err
 	}
 
+	if kcp.Spec.K0sConfigSpec.Tunneling.Enabled {
+		if kcp.Spec.K0sConfigSpec.Tunneling.Mode == "proxy" {
+			secretName := secret.Name(cluster.Name+"-proxied", secret.Kubeconfig)
+			err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, &corev1.Secret{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					kc, err := c.generateKubeconfig(ctx, cluster, fmt.Sprintf("https://%s", cluster.Spec.ControlPlaneEndpoint.String()))
+					if err != nil {
+						return err
+					}
+
+					for cn := range kc.Clusters {
+						kc.Clusters[cn].ProxyURL = fmt.Sprintf("http://%s:%d", kcp.Spec.K0sConfigSpec.Tunneling.ServerAddress, kcp.Spec.K0sConfigSpec.Tunneling.TunnelingNodePort)
+					}
+
+					err = c.createKubeconfigSecret(ctx, kc, cluster, secretName)
+					if err != nil {
+						return err
+					}
+				}
+				return err
+			}
+		} else {
+			secretName := secret.Name(cluster.Name+"-tunneled", secret.Kubeconfig)
+			err := c.Client.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: secretName}, &corev1.Secret{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					kc, err := c.generateKubeconfig(ctx, cluster, fmt.Sprintf("https://%s:%d", kcp.Spec.K0sConfigSpec.Tunneling.ServerAddress, kcp.Spec.K0sConfigSpec.Tunneling.TunnelingNodePort))
+					if err != nil {
+						return err
+					}
+
+					err = c.createKubeconfigSecret(ctx, kc, cluster, secretName)
+					if err != nil {
+						return err
+					}
+				}
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 
 func (c *K0sController) reconcile(ctx context.Context, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (ctrl.Result, error) {
-	err := c.reconcileKubeconfig(ctx, cluster)
+	err := c.reconcileKubeconfig(ctx, cluster, kcp)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("error reconciling kubeconfig secret: %w", err)
 	}
@@ -276,12 +318,24 @@ func (c *K0sController) reconcileTunneling(ctx context.Context, cluster *cluster
 		return fmt.Errorf("error creating FRP token secret: %w", err)
 	}
 
-	frpsConfig := `
+	var frpsConfig string
+	if kcp.Spec.K0sConfigSpec.Tunneling.Mode == "proxy" {
+		frpsConfig = `
+[common]
+bind_port = 7000
+tcpmux_httpconnect_port = 6443
+authentication_method = token
+token = ` + frpToken + `
+`
+	} else {
+		frpsConfig = `
 [common]
 bind_port = 7000
 authentication_method = token
 token = ` + frpToken + `
 `
+	}
+
 	frpsCMName := kcp.GetName() + "-frps-config"
 	cm := corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{

internal/controller/controlplane/util.go

@@ -2,11 +2,21 @@ package controlplane
 
 import (
 	"context"
+	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util"
 
-	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
-
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/cluster-api/util/certs"
+	"sigs.k8s.io/cluster-api/util/kubeconfig"
+	"sigs.k8s.io/cluster-api/util/secret"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
 )
 
 func (c *K0sController) getMachineTemplate(ctx context.Context, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
@@ -25,3 +35,56 @@ func (c *K0sController) getMachineTemplate(ctx context.Context, kcp *cpv1beta1.K
 	}
 	return machineTemplate, nil
 }
+
+func (c *K0sController) generateKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, endpoint string) (*api.Config, error) {
+	clusterName := util.ObjectKey(cluster)
+	clusterCA, err := secret.GetFromNamespacedName(ctx, c.Client, clusterName, secret.ClusterCA)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, kubeconfig.ErrDependentCertificateNotFound
+		}
+		return nil, err
+	}
+
+	cert, err := certs.DecodeCertPEM(clusterCA.Data[secret.TLSCrtDataName])
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode CA Cert: %w", err)
+	} else if cert == nil {
+		return nil, fmt.Errorf("certificate not found in config: %w", err)
+	}
+
+	key, err := certs.DecodePrivateKeyPEM(clusterCA.Data[secret.TLSKeyDataName])
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode private key: %w", err)
+	} else if key == nil {
+		return nil, fmt.Errorf("CA private key not found: %w", err)
+	}
+
+	cfg, err := kubeconfig.New(clusterName.Name, endpoint, cert, key)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate a kubeconfig: %w", err)
+	}
+
+	return cfg, nil
+
+}
+
+func (c *K0sController) createKubeconfigSecret(ctx context.Context, cfg *api.Config, cluster *clusterv1.Cluster, secretName string) error {
+	cfgBytes, err := clientcmd.Write(*cfg)
+	if err != nil {
+		return fmt.Errorf("failed to serialize config to yaml: %w", err)
+	}
+
+	clusterName := util.ObjectKey(cluster)
+	owner := metav1.OwnerReference{
+		APIVersion: clusterv1.GroupVersion.String(),
+		Kind:       "Cluster",
+		Name:       cluster.Name,
+		UID:        cluster.UID,
+	}
+
+	kcSecret := kubeconfig.GenerateSecretWithOwner(clusterName, cfgBytes, owner)
+	kcSecret.Name = secretName
+
+	return c.Create(ctx, kcSecret)
+}

inttest/Makefile.variables

@@ -15,5 +15,6 @@ smoketests := \
     check-capi-controlplane-docker-downscaling \
     check-capi-controlplane-docker-worker \
     check-capi-controlplane-docker-tunneling \
+    check-capi-controlplane-docker-tunneling-proxy \
     check-monitoring \
     check-capi-docker-machinedeployment \