Implement keepalived load balancer

cmd/controller/certificates.go

@@ -182,6 +182,20 @@ func (c *Certificates) Init(ctx context.Context) error {
 	hostnames = append(hostnames, localIPs...)
 	hostnames = append(hostnames, c.ClusterSpec.API.Sans()...)
 
+	// Add to SANs the IPs from the control plane load balancer
+	cplb := c.ClusterSpec.Network.ControlPlaneLoadBalancing
+	if cplb != nil && cplb.Enabled && cplb.Keepalived != nil {
+		for _, v := range cplb.Keepalived.VRRPInstances {
+			for _, vip := range v.VirtualIPs {
+				ip, _, err := net.ParseCIDR(vip)
+				if err != nil {
+					return fmt.Errorf("error parsing virtualIP %s: %w", vip, err)
+				}
+				hostnames = append(hostnames, ip.String())
+			}
+		}
+	}
+
 	internalAPIAddress, err := c.ClusterSpec.Network.InternalAPIAddresses()
 	if err != nil {
 		return err

cmd/controller/controller.go

@@ -235,8 +235,10 @@ func (c *command) start(ctx context.Context) error {
 
 		nodeComponents.Add(ctx, &controller.Keepalived{
 			K0sVars:         c.K0sVars,
-			Config:          cplb,
+			Config:          cplb.Keepalived,
 			DetailedLogging: c.Debug,
+			KubeConfigPath:  c.K0sVars.AdminKubeConfigPath,
+			APIPort:         nodeConfig.Spec.API.Port,
 		})
 	}
 

docs/configuration.md

@@ -304,26 +304,47 @@ node-local load balancing.
 
 Configuration options related to k0s's [control plane load balancing] feature
 
-| Element         | Description                                                                                                 |
-| --------------- | ----------------------------------------------------------------------------------------------------------- |
-| `enabled`       | Indicates if control plane load balancing should be enabled. Default: `false`.                              |
-| `vrrpInstances` | Configuration options related to the VRRP. This is an array which allows to configure multiple virtual IPs. |
+| Element      | Description                                                                                 |
+| ------------ | ------------------------------------------------------------------------------------------- |
+| `enabled`    | Indicates if control plane load balancing should be enabled. Default: `false`.              |
+| `type`       | Indicates the backend for CPLB. If this isn't defined to `Keepalived`, CPLB will not start. |
+| `keepalived` | Contains the keepalived configuration.                                                      |
 
 [control plane load balancing]: cplb.md
 
-##### `spec.network.controlPlaneLoadBalancing.VRRPInstances`
+##### `spec.network.controlPlaneLoadBalancing.Keepalived`
+
+Configuration options related to keepalived in [control plane load balancing]
+
+| Element          | Description                                                                                                 |
+| ---------------- | ----------------------------------------------------------------------------------------------------------- |
+| `vrrpInstances`  | Configuration options related to the VRRP. This is an array which allows to configure multiple virtual IPs. |
+| `virtualServers` | Configuration options related LoadBalancing. This is an array which allows to configure multiple LBs.       |
+
+##### `spec.network.controlPlaneLoadBalancing.keepalived.vrrpInstances`
 
 Configuration options required for using VRRP to configure VIPs in control plane load balancing.
 
 | Element           | Description                                                                                                       |
 | ----------------- | ----------------------------------------------------------------------------------------------------------------- |
-| `name`            | The name of the VRRP instance. If omitted it generates a predictive name shared across all nodes.                 |
 | `virtualIPs`      | A list of the CIDRs handled by the VRRP instance.                                                                 |
 | `interface`       | The interface used by each VRRPInstance. If undefined k0s will try to auto detect it based on the default gateway |
 | `virtualRouterId` | Virtual router ID for the instance. Default: `51`                                                                 |
 | `advertInterval`  | Advertisement interval in seconds. Default: `1`.                                                                  |
 | `authPass`        | The password used for accessing vrrpd. This field is mandatory and must be under 8 characters long                |
 
+##### `spec.network.controlPlaneLoadBalancing.keepalived.virtualServers`
+
+Configuration options required for using VRRP to configure VIPs in control plane load balancing.
+
+| Element                     | Description                                                                                                                                      |
+| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `ipAddress`                 | The load balancer's listen address.                                                                                                              |
+| `delayLoop`                 | Delay timer for check polling. DelayLoop accepts microsecond precision. Further precision will be truncated without warnings. Example: `10s`.    |
+| `lbAlgo`                    | Algorithm used by keepalived. Supported algorithms: `rr`, `wrr`, `lc`, `wlc`, `lblc`, `dh`, `sh`, `sed`, `nq`. Default: `rr`.                    |
+| `lbKind`                    | Kind of ipvs load balancer. Supported values: `NAT`, `DR`, `TUN`  Default: `DR`.                                                                 |
+| `persistenceTimeoutSeconds` | Timeout value for persistent connections in seconds. Must be in the range of 1-2678400 (31 days). If not specified, defaults to 360 (6 minutes). |
+
 ### `spec.controllerManager`
 
 | Element     | Description                                                                                                             |

docs/cplb.md

@@ -3,17 +3,23 @@
 For clusters that don't have an [externally managed load balancer](high-availability.md#load-balancer) for the k0s
 control plane, there is another option to get a highly available control plane called control plane load balancing (CPLB).
 
-CPLB allows automatic assigned of predefined IP addresses using VRRP across masters.
+CPLB has two features that are independent, but normally will be used together: VRRP Instances, which allows
+automatic assignation of predefined IP addresses using VRRP across control plane nodes. VirtualServers allows to
+do Load Balancing to the other control plane nodes.
+
+This feature is intended to be used for external traffic. This feature is fully compatible with
+[node-local load balancing (NLLB)](nllb.md) which means CPLB can be used for external traffic and NLLB for
+internal traffic at the same time.
 
 ## Technical functionality
 
-The k0s control plane load balancer provides k0s with virtual IPs on each
-controller node. This allows the control plane to be highly available using
-VRRP (Virtual Router Redundancy Protocol) as long as the network
-infrastructure allows multicast and GARP.
+The k0s control plane load balancer provides k0s with virtual IPs and TCP
+load Balancing on each controller node. This allows the control plane to
+be highly available using VRRP (Virtual Router Redundancy Protocol) and
+IPVS long as the network infrastructure allows multicast and GARP.
 
 [Keepalived](https://www.keepalived.org/) is the only load balancer that is
-supported so far and currently there are no plans to support other alternatives.
+supported so far. Currently there are no plans to support other alternatives.
 
 ## VRRP Instances
 
@@ -46,19 +52,24 @@ following:
   These do not provide any sort of security against ill-intentioned attacks, they are
   safety features to prevent accidental conflicts between VRRP instances in the same
   network segment.
+* If `VirtualServers` are used, the cluster configuration mustn't specify a non-empty
+  [`spec.api.externalAddress`][specapi]. If only `VRRPInstances` are specified, a
+  non-empty [`spec.api.externalAddress`][specapi] may be specified.
 
 Add the following to the cluster configuration (`k0s.yaml`):
 
 ```yaml
 spec:
-  api:
-    externalAddress: <External address> # This isn't a requirement, but it's a common use case.
   network:
     controlPlaneLoadBalancing:
       enabled: true
-      vrrpInstances:
-      - virtualIPs: ["<External address IP>/<external address IP netmask"]
-        authPass: <password>
+      type: Keepalived
+      keepalived:
+        vrrpInstances:
+        - virtualIPs: ["<External address IP>/<external address IP netmask"]
+          authPass: <password>
+        virtualServers:
+        - ipAddress: "ipAddress"
 ```
 
 Or alternatively, if using [`k0sctl`](k0sctl-install.md), add the following to
@@ -69,24 +80,28 @@ spec:
   k0s:
     config:
       spec:
-        api:
-          externalAddress: <External address> # This isn't a requirement, but it's a common use case.
         network:
           controlPlaneLoadBalancing:
             enabled: true
-            vrrpInstances:
-            - virtualIPs: ["<External address IP>/<external address IP netmask>"]
-              authPass: <password>
+            type: Keepalived
+            keepalived:
+              vrrpInstances:
+              - virtualIPs: ["<External address IP>/<external address IP netmask>"]
+                authPass: <password>
+              virtualServers:
+              - ipAddress: "<External ip address>"
 ```
 
 Because this is a feature intended to configure the apiserver, CPLB noes not
 support dynamic configuration and in order to make changes you need to restart
 the k0s controllers to make changes.
 
+[specapi]: configuration.md#specapi
+
 ## Full example using `k0sctl`
 
 The following example shows a full `k0sctl` configuration file featuring three
-controllers and three workers with control plane load balancing enabled:
+controllers and three workers with control plane load balancing enabled.
 
 ```yaml
 apiVersion: k0sctl.k0sproject.io/v1beta1
@@ -142,13 +157,18 @@ spec:
     config:
       spec:
         api:
-          externalAddress: 192.168.122.200
+          sans:
+          - 192.168.122.200
         network:
           controlPlaneLoadBalancing:
             enabled: true
-            vrrpInstances:
-            - virtualIPs: ["192.168.122.200/24"]
-              authPass: Example
+            type: Keepalived:
+            keepalived:
+              vrrpInstances:
+              - virtualIPs: ["192.168.122.200/24"]
+                authPass: Example
+              virtualServers:
+              - ipAddress: "<External ip address>"
 ```
 
 Save the above configuration into a file called `k0sctl.yaml` and apply it in
@@ -319,6 +339,15 @@ controller-1
 controller-2
 2: eth0    inet 192.168.122.87/24 brd 192.168.122.255 scope global dynamic noprefixroute eth0\       valid_lft 2182sec preferred_lft 2182sec
 3: dummyvip0    inet 192.168.122.200/32 scope global dummyvip0\       valid_lft forever preferred_lft forever
+
+$ for i in controller-{0..2} ; do echo $i ; ipvsadm --save -n; done
+IP Virtual Server version 1.2.1 (size=4096)
+Prot LocalAddress:Port Scheduler Flags
+  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
+TCP  192.168.122.200:6443 rr persistent 360
+  -> 192.168.122.185:6443              Route   1      0          0
+  -> 192.168.122.87:6443               Route   1      0          0
+  -> 192.168.122.122:6443              Route   1      0          0
 ````
 
 And the cluster will be working normally:

inttest/bootloose-alpine/Dockerfile

@@ -21,7 +21,8 @@ RUN apk add --no-cache \
   curl \
   haproxy \
   nginx \
-  inotify-tools
+  inotify-tools \
+  ipvsadm
 # enable syslog and sshd
 RUN rc-update add syslog boot
 RUN rc-update add sshd default

inttest/cplb/cplb_test.go

@@ -33,26 +33,31 @@ type keepalivedSuite struct {
 
 const haControllerConfig = `
 spec:
-  api:
-    externalAddress: %s
   network:
     controlPlaneLoadBalancing:
       enabled: true
-      vrrpInstances:
-      - virtualIPs: ["%s/24"]
-        authPass: "123456"
+      type: Keepalived
+      keepalived:
+        vrrpInstances:
+        - virtualIPs: ["%s/16"]
+          authPass: "123456"
+        virtualServers:
+        - ipAddress: %s
+    nodeLocalLoadBalancing:
+      enabled: true
+      type: EnvoyProxy
 `
 
 // SetupTest prepares the controller and filesystem, getting it into a consistent
 // state which we can run tests against.
 func (s *keepalivedSuite) TestK0sGetsUp() {
-	ipAddress := s.getLBAddress()
+	lb := s.getLBAddress()
 	ctx := s.Context()
 	var joinToken string
 
 	for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ {
 		s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second))
-		s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, ipAddress, ipAddress))
+		s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb))
 
 		// Note that the token is intentionally empty for the first controller
 		s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken))
@@ -85,18 +90,22 @@ func (s *keepalivedSuite) TestK0sGetsUp() {
 
 	// Verify that all servers have the dummy interface
 	for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ {
-		s.checkDummy(ctx, s.ControllerNode(idx), ipAddress)
+		s.checkDummy(ctx, s.ControllerNode(idx), lb)
 	}
 
 	// Verify that only one controller has the VIP in eth0
 	count := 0
 	for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ {
-		if s.hasVIP(ctx, s.ControllerNode(idx), ipAddress) {
+		if s.hasVIP(ctx, s.ControllerNode(idx), lb) {
 			count++
 		}
 	}
-	s.Require().Equal(1, count, "Expected only one controller to have the VIP")
+	s.Require().Equal(1, count, "Expected exactly one controller to have the VIP")
 
+	// Verify that the real servers are present in the ipvsadm output
+	for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ {
+		s.validateRealServers(ctx, s.ControllerNode(idx), lb)
+	}
 }
 
 // getLBAddress returns the IP address of the controller 0 and it adds 100 to
@@ -119,6 +128,27 @@ func (s *keepalivedSuite) getLBAddress() string {
 	return fmt.Sprintf("%s.%d", strings.Join(parts[:3], "."), lastOctet)
 }
 
+// validateRealServers checks that the real servers are present in the
+// ipvsadm output.
+func (s *keepalivedSuite) validateRealServers(ctx context.Context, node string, vip string) {
+	ssh, err := s.SSH(ctx, node)
+	s.Require().NoError(err)
+	defer ssh.Disconnect()
+
+	servers := []string{}
+	for i := 0; i < s.BootlooseSuite.ControllerCount; i++ {
+		servers = append(servers, s.GetIPAddress(s.ControllerNode(i)))
+	}
+
+	output, err := ssh.ExecWithOutput(ctx, "ipvsadm --save -n")
+	s.Require().NoError(err)
+
+	for _, server := range servers {
+		s.Require().Contains(output, fmt.Sprintf("-a -t %s:6443 -r %s", vip, server), "Controller %s is missing a server in ipvsadm", node)
+	}
+
+}
+
 // checkDummy checks that the dummy interface is present on the given node and
 // that it has only the virtual IP address.
 func (s *keepalivedSuite) checkDummy(ctx context.Context, node string, vip string) {
@@ -145,7 +175,7 @@ func (s *keepalivedSuite) hasVIP(ctx context.Context, node string, vip string) b
 	output, err := ssh.ExecWithOutput(ctx, "ip --oneline addr show eth0")
 	s.Require().NoError(err)
 
-	return strings.Contains(output, fmt.Sprintf("inet %s/24", vip))
+	return strings.Contains(output, fmt.Sprintf("inet %s/16", vip))
 }
 
 // TestKeepAlivedSuite runs the keepalived test suite. It verifies that the

pkg/apis/k0s/v1beta1/clusterconfig_types.go

@@ -335,6 +335,12 @@ func (s *ClusterSpec) Validate() (errs []error) {
 		errs = append(errs, err)
 	}
 
+	if s.Network != nil && s.Network.ControlPlaneLoadBalancing != nil {
+		for _, err := range s.Network.ControlPlaneLoadBalancing.Validate(s.API.ExternalAddress) {
+			errs = append(errs, fmt.Errorf("controlPlaneLoadBalancing: %w", err))
+		}
+	}
+
 	return
 }
 
@@ -390,6 +396,7 @@ func (c *ClusterConfig) Validate() (errs []error) {
 // - StorageSpec
 // - Network.ServiceCIDR
 // - Network.ClusterDomain
+// - Network.ControlPlaneLoadBalancing
 // - Install
 func (c *ClusterConfig) GetClusterWideConfig() *ClusterConfig {
 	c = c.DeepCopy()
@@ -399,6 +406,7 @@ func (c *ClusterConfig) GetClusterWideConfig() *ClusterConfig {
 		if c.Spec.Network != nil {
 			c.Spec.Network.ServiceCIDR = ""
 			c.Spec.Network.ClusterDomain = ""
+			c.Spec.Network.ControlPlaneLoadBalancing = nil
 		}
 		c.Spec.Install = nil
 	}