Client connection tunneling prototype

Signed-off-by: Alexey Makhov <[email protected]>

.github/workflows/go.yml

@@ -106,6 +106,7 @@ jobs:
           - check-capi-docker-machinedeployment
           - check-capi-controlplane-docker
           - check-capi-controlplane-docker-downscaling
+          - check-capi-controlplane-docker-tunneling
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3

api/bootstrap/v1beta1/k0s_types.go

@@ -173,4 +173,15 @@ type K0sConfigSpec struct {
 	// If the version field is specified, it is ignored, and whatever version is downloaded from the URL is used.
 	// +kubebuilder:validation:Optional
 	DownloadURL string `json:"downloadURL,omitempty"`
+
+	// Tunneling defines the tunneling configuration for the cluster.
+	//+kubebuilder:validation:Optional
+	Tunneling TunnelingSpec `json:"tunneling,omitempty"`
+}
+
+type TunnelingSpec struct {
+	// Enabled specifies whether tunneling is enabled.
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=false
+	Enabled bool `json:"enabled,omitempty"`
 }

api/controlplane/v1beta1/k0s_types.go

@@ -35,8 +35,8 @@ func init() {
 type K0sControlPlane struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-	Spec              K0sControlPlaneSpec `json:"spec,omitempty"`
 
+	Spec   K0sControlPlaneSpec   `json:"spec,omitempty"`
 	Status K0sControlPlaneStatus `json:"status,omitempty"`
 }
 

config/crd/bases/bootstrap.cluster.x-k8s.io_k0scontrollerconfigs.yaml

@@ -80,6 +80,15 @@ spec:
                 items:
                   type: string
                 type: array
+              tunneling:
+                description: Tunneling defines the tunneling configuration for the
+                  cluster.
+                properties:
+                  enabled:
+                    default: false
+                    description: Enabled specifies whether tunneling is enabled.
+                    type: boolean
+                type: object
               version:
                 description: 'Version is the version of k0s to use. In case this is
                   not set, the latest version is used. Make sure the version is compatible

config/crd/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml

@@ -82,6 +82,15 @@ spec:
                     items:
                       type: string
                     type: array
+                  tunneling:
+                    description: Tunneling defines the tunneling configuration for
+                      the cluster.
+                    properties:
+                      enabled:
+                        default: false
+                        description: Enabled specifies whether tunneling is enabled.
+                        type: boolean
+                    type: object
                   version:
                     description: 'Version is the version of k0s to use. In case this
                       is not set, the latest version is used. Make sure the version

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:

config/samples/capi/docker/kind.yaml

@@ -6,4 +6,7 @@ nodes:
 - role: control-plane
   extraMounts:
     - hostPath: /var/run/docker.sock
-      containerPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
+  extraPortMappings:
+    - containerPort: 31443
+      hostPort: 31443

internal/controller/bootstrap/controlplane_bootstrap_controller.go

@@ -63,6 +63,7 @@ const joinTokenFilePath = "/etc/k0s.token"
 // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 
 func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
 	log := log.FromContext(ctx).WithValues("K0sControllerConfig", req.NamespacedName)
@@ -171,6 +172,9 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
 		}
 		installCmd = createCPInstallCmdWithJoinToken(config, joinTokenFilePath)
 	}
+	if config.Spec.Tunneling.Enabled {
+		files = append(files, c.genTunnelingFiles(ctx, scope, config)...)
+	}
 	files = append(files, config.Spec.Files...)
 
 	downloadCommands := createCPDownloadCommands(config)
@@ -300,6 +304,65 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
 	return files, err
 }
 
+func (c *ControlPlaneController) genTunnelingFiles(_ context.Context, _ *Scope, _ *bootstrapv1.K0sControllerConfig) []cloudinit.File {
+	tunnelingResources := `
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: frpc-config
+  namespace: kube-system
+data:
+  frpc.ini: |
+    [common]
+    server_addr = 172.18.0.2
+    server_port = 31700
+    
+    [kube-apiserver]
+    type = tcp
+    local_ip = 10.96.0.1
+    local_port = 443
+    remote_port = 6443
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frpc
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: frpc
+  template:
+    metadata:
+      labels:
+        app: frpc
+    spec:
+      containers:
+        - name: frpc
+          image: snowdreamtech/frpc:0.51.3
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: frpc-config
+              mountPath: /etc/frp/frpc.ini
+              subPath: frpc.ini
+      volumes:
+        - name: frpc-config
+          configMap:
+            name: frpc-config
+            items:
+              - key: frpc.ini
+                path: frpc.ini
+
+`
+	return []cloudinit.File{{
+		Path:        "/var/lib/k0s/manifests/k0smotron-tunneling/manifest.yaml",
+		Permissions: "0644",
+		Content:     tunnelingResources,
+	}}
+}
+
 func (c *ControlPlaneController) getCerts(ctx context.Context, scope *Scope) ([]cloudinit.File, *secret.Certificate, error) {
 	var files []cloudinit.File
 	certificates := secret.NewCertificatesForInitialControlPlane(&kubeadmbootstrapv1.ClusterConfiguration{

internal/controller/controlplane/helper.go

@@ -3,6 +3,7 @@ package controlplane
 import (
 	"context"
 	"fmt"
+	"k8s.io/utils/pointer"
 	"strings"
 
 	"github.com/Masterminds/semver"
@@ -74,31 +75,32 @@ func (c *K0sController) generateMachine(_ context.Context, name string, cluster
 	}
 }
 
-func (c *K0sController) createMachineFromTemplate(ctx context.Context, name string, _ *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
-	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, kcp)
+func (c *K0sController) createMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
+	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, cluster, kcp)
 	if err != nil {
 		return nil, err
 	}
 
 	if err = c.Client.Patch(ctx, machineFromTemplate, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",
+		Force:        pointer.Bool(true),
 	}); err != nil {
 		return nil, err
 	}
 
 	return machineFromTemplate, nil
 }
 
-func (c *K0sController) deleteMachineFromTemplate(ctx context.Context, name string, kcp *cpv1beta1.K0sControlPlane) error {
-	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, kcp)
+func (c *K0sController) deleteMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) error {
+	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, cluster, kcp)
 	if err != nil {
 		return err
 	}
 
 	return c.Client.Delete(ctx, machineFromTemplate)
 }
 
-func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name string, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
+func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
 	unstructuredMachineTemplate, err := c.getMachineTemplate(ctx, kcp)
 	if err != nil {
 		return nil, err
@@ -130,7 +132,7 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		labels[k] = v
 	}
 
-	labels[clusterv1.ClusterNameLabel] = kcp.Name
+	labels[clusterv1.ClusterNameLabel] = cluster.GetName()
 	labels[clusterv1.MachineControlPlaneLabel] = ""
 	labels[clusterv1.MachineControlPlaneNameLabel] = kcp.Name
 	machine.SetLabels(labels)