Merge pull request #258 from makhov/frp-tunneling

Client connection tunneling

.github/workflows/go.yml

@@ -106,6 +106,8 @@ jobs:
           - check-capi-docker-machinedeployment
           - check-capi-controlplane-docker
           - check-capi-controlplane-docker-downscaling
+          - check-capi-controlplane-docker-tunneling
+          - check-capi-controlplane-docker-tunneling-proxy
           - check-capi-controlplane-docker-worker
     steps:
       - name: Check out code into the Go module directory
@@ -158,4 +160,5 @@ jobs:
         run: |
           kind get kubeconfig > kind.conf
           export KUBECONFIG=$(realpath kind.conf)
-          KEEP_AFTER_TEST=true make -C inttest ${{ matrix.smoke-suite }}
+          docker system prune -f
+          make -C inttest ${{ matrix.smoke-suite }}

api/bootstrap/v1beta1/k0s_types.go

@@ -173,4 +173,34 @@ type K0sConfigSpec struct {
 	// If the version field is specified, it is ignored, and whatever version is downloaded from the URL is used.
 	// +kubebuilder:validation:Optional
 	DownloadURL string `json:"downloadURL,omitempty"`
+
+	// Tunneling defines the tunneling configuration for the cluster.
+	//+kubebuilder:validation:Optional
+	Tunneling TunnelingSpec `json:"tunneling,omitempty"`
+}
+
+type TunnelingSpec struct {
+	// Enabled specifies whether tunneling is enabled.
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=false
+	Enabled bool `json:"enabled,omitempty"`
+	// Server address of the tunneling server.
+	// If empty, k0smotron will try to detect worker node address for.
+	//+kubebuilder:validation:Optional
+	ServerAddress string `json:"serverAddress,omitempty"`
+	// NodePort to publish for server port of the tunneling server.
+	// If empty, k0smotron will use the default one.
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=31700
+	ServerNodePort int32 `json:"serverNodePort,omitempty"`
+	// NodePort to publish for tunneling port.
+	// If empty, k0smotron will use the default one.
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=31443
+	TunnelingNodePort int32 `json:"tunnelingNodePort,omitempty"`
+	// Mode describes tunneling mode.
+	// If empty, k0smotron will use the default one.
+	//+kubebuilder:validation:Enum=tunnel;proxy
+	//+kubebuilder:default=tunnel
+	Mode string `json:"mode,omitempty"`
 }

api/controlplane/v1beta1/k0s_types.go

@@ -35,8 +35,8 @@ func init() {
 type K0sControlPlane struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-	Spec              K0sControlPlaneSpec `json:"spec,omitempty"`
 
+	Spec   K0sControlPlaneSpec   `json:"spec,omitempty"`
 	Status K0sControlPlaneStatus `json:"status,omitempty"`
 }
 

config/crd/bases/bootstrap.cluster.x-k8s.io_k0scontrollerconfigs.yaml

@@ -80,6 +80,39 @@ spec:
                 items:
                   type: string
                 type: array
+              tunneling:
+                description: Tunneling defines the tunneling configuration for the
+                  cluster.
+                properties:
+                  enabled:
+                    default: false
+                    description: Enabled specifies whether tunneling is enabled.
+                    type: boolean
+                  mode:
+                    default: tunnel
+                    description: Mode describes tunneling mode. If empty, k0smotron
+                      will use the default one.
+                    enum:
+                    - tunnel
+                    - proxy
+                    type: string
+                  serverAddress:
+                    description: Server address of the tunneling server. If empty,
+                      k0smotron will try to detect worker node address for.
+                    type: string
+                  serverNodePort:
+                    default: 31700
+                    description: NodePort to publish for server port of the tunneling
+                      server. If empty, k0smotron will use the default one.
+                    format: int32
+                    type: integer
+                  tunnelingNodePort:
+                    default: 31443
+                    description: NodePort to publish for tunneling port. If empty,
+                      k0smotron will use the default one.
+                    format: int32
+                    type: integer
+                type: object
               version:
                 description: 'Version is the version of k0s to use. In case this is
                   not set, the latest version is used. Make sure the version is compatible

config/crd/bases/controlplane.cluster.x-k8s.io_k0scontrolplanes.yaml

@@ -82,6 +82,39 @@ spec:
                     items:
                       type: string
                     type: array
+                  tunneling:
+                    description: Tunneling defines the tunneling configuration for
+                      the cluster.
+                    properties:
+                      enabled:
+                        default: false
+                        description: Enabled specifies whether tunneling is enabled.
+                        type: boolean
+                      mode:
+                        default: tunnel
+                        description: Mode describes tunneling mode. If empty, k0smotron
+                          will use the default one.
+                        enum:
+                        - tunnel
+                        - proxy
+                        type: string
+                      serverAddress:
+                        description: Server address of the tunneling server. If empty,
+                          k0smotron will try to detect worker node address for.
+                        type: string
+                      serverNodePort:
+                        default: 31700
+                        description: NodePort to publish for server port of the tunneling
+                          server. If empty, k0smotron will use the default one.
+                        format: int32
+                        type: integer
+                      tunnelingNodePort:
+                        default: 31443
+                        description: NodePort to publish for tunneling port. If empty,
+                          k0smotron will use the default one.
+                        format: int32
+                        type: integer
+                    type: object
                   version:
                     description: 'Version is the version of k0s to use. In case this
                       is not set, the latest version is used. Make sure the version

config/rbac/role.yaml

@@ -18,6 +18,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - apps
   resources:

config/samples/capi/docker/kind.yaml

@@ -6,4 +6,7 @@ nodes:
 - role: control-plane
   extraMounts:
     - hostPath: /var/run/docker.sock
-      containerPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
+  extraPortMappings:
+    - containerPort: 31443
+      hostPort: 31443

internal/controller/bootstrap/controlplane_bootstrap_controller.go

@@ -63,6 +63,7 @@ const joinTokenFilePath = "/etc/k0s.token"
 // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machines;machines/status,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=exp.cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 
 func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
 	log := log.FromContext(ctx).WithValues("K0sControllerConfig", req.NamespacedName)
@@ -136,12 +137,23 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
 	)
 
 	if config.Spec.K0s != nil {
-		//config.Spec.K0s.SetUnstructuredContent(map["spec"]interface{}{})
 		err = unstructured.SetNestedField(config.Spec.K0s.Object, scope.Cluster.Spec.ControlPlaneEndpoint.Host, "spec", "api", "externalAddress")
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error setting control plane endpoint: %v", err)
 		}
 
+		if config.Spec.Tunneling.ServerAddress != "" {
+			sans, _, err := unstructured.NestedSlice(config.Spec.K0s.Object, "spec", "api", "sans")
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("error getting sans from config: %v", err)
+			}
+			sans = append(sans, config.Spec.Tunneling.ServerAddress)
+			err = unstructured.SetNestedSlice(config.Spec.K0s.Object, sans, "spec", "api", "sans")
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("error setting sans to the config: %v", err)
+			}
+		}
+
 		k0sConfigBytes, err := config.Spec.K0s.MarshalJSON()
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("error marshalling k0s config: %v", err)
@@ -171,6 +183,13 @@ func (c *ControlPlaneController) Reconcile(ctx context.Context, req ctrl.Request
 		}
 		installCmd = createCPInstallCmdWithJoinToken(config, joinTokenFilePath)
 	}
+	if config.Spec.Tunneling.Enabled {
+		tunnelingFiles, err := c.genTunnelingFiles(ctx, scope, config)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("error generating tunneling files: %v", err)
+		}
+		files = append(files, tunnelingFiles...)
+	}
 	files = append(files, config.Spec.Files...)
 
 	downloadCommands := createCPDownloadCommands(config)
@@ -300,6 +319,88 @@ func (c *ControlPlaneController) genControlPlaneJoinFiles(ctx context.Context, s
 	return files, err
 }
 
+func (c *ControlPlaneController) genTunnelingFiles(ctx context.Context, scope *Scope, kcs *bootstrapv1.K0sControllerConfig) ([]cloudinit.File, error) {
+	secretName := scope.Cluster.Name + "-frp-token"
+	frpSecret := corev1.Secret{}
+	err := c.Client.Get(ctx, client.ObjectKey{Namespace: scope.Cluster.Namespace, Name: secretName}, &frpSecret)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get frp secret: %w", err)
+	}
+	frpToken := string(frpSecret.Data["value"])
+
+	var modeConfig string
+	if kcs.Spec.Tunneling.Mode == "proxy" {
+		modeConfig = fmt.Sprintf(`
+    type = tcpmux
+    custom_domains = %s
+    multiplexer = httpconnect
+`, scope.Cluster.Spec.ControlPlaneEndpoint.Host)
+	} else {
+		modeConfig = `
+    remote_port = 6443
+`
+	}
+
+	tunnelingResources := `
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: frpc-config
+  namespace: kube-system
+data:
+  frpc.ini: |
+    [common]
+    authentication_method = token
+    server_addr = %s
+    server_port = %d
+    token = %s
+    
+    [kube-apiserver]
+    type = tcp
+    local_ip = 10.96.0.1
+    local_port = 443
+    %s
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frpc
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: frpc
+  template:
+    metadata:
+      labels:
+        app: frpc
+    spec:
+      containers:
+        - name: frpc
+          image: snowdreamtech/frpc:0.51.3
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: frpc-config
+              mountPath: /etc/frp/frpc.ini
+              subPath: frpc.ini
+      volumes:
+        - name: frpc-config
+          configMap:
+            name: frpc-config
+            items:
+              - key: frpc.ini
+                path: frpc.ini
+
+`
+	return []cloudinit.File{{
+		Path:        "/var/lib/k0s/manifests/k0smotron-tunneling/manifest.yaml",
+		Permissions: "0644",
+		Content:     fmt.Sprintf(tunnelingResources, kcs.Spec.Tunneling.ServerAddress, kcs.Spec.Tunneling.ServerNodePort, frpToken, modeConfig),
+	}}, nil
+}
+
 func (c *ControlPlaneController) getCerts(ctx context.Context, scope *Scope) ([]cloudinit.File, *secret.Certificate, error) {
 	var files []cloudinit.File
 	certificates := secret.NewCertificatesForInitialControlPlane(&kubeadmbootstrapv1.ClusterConfiguration{

internal/controller/controlplane/helper.go

@@ -74,8 +74,8 @@ func (c *K0sController) generateMachine(_ context.Context, name string, cluster
 	}
 }
 
-func (c *K0sController) createMachineFromTemplate(ctx context.Context, name string, _ *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
-	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, kcp)
+func (c *K0sController) createMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
+	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, cluster, kcp)
 	if err != nil {
 		return nil, err
 	}
@@ -89,16 +89,16 @@ func (c *K0sController) createMachineFromTemplate(ctx context.Context, name stri
 	return machineFromTemplate, nil
 }
 
-func (c *K0sController) deleteMachineFromTemplate(ctx context.Context, name string, kcp *cpv1beta1.K0sControlPlane) error {
-	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, kcp)
+func (c *K0sController) deleteMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) error {
+	machineFromTemplate, err := c.generateMachineFromTemplate(ctx, name, cluster, kcp)
 	if err != nil {
 		return err
 	}
 
 	return c.Client.Delete(ctx, machineFromTemplate)
 }
 
-func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name string, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
+func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name string, cluster *clusterv1.Cluster, kcp *cpv1beta1.K0sControlPlane) (*unstructured.Unstructured, error) {
 	unstructuredMachineTemplate, err := c.getMachineTemplate(ctx, kcp)
 	if err != nil {
 		return nil, err
@@ -130,7 +130,7 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		labels[k] = v
 	}
 
-	labels[clusterv1.ClusterNameLabel] = kcp.Name
+	labels[clusterv1.ClusterNameLabel] = cluster.GetName()
 	labels[clusterv1.MachineControlPlaneLabel] = ""
 	labels[clusterv1.MachineControlPlaneNameLabel] = kcp.Name
 	machine.SetLabels(labels)