From efb3387222edcf72586ebfaa6375156f1fdceda0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 29 Apr 2024 13:22:07 +0200 Subject: [PATCH 01/12] Add API types for CPLB virtualServers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- pkg/apis/k0s/v1beta1/cplb.go | 119 ++++++++++++++++-- pkg/apis/k0s/v1beta1/cplb_test.go | 116 +++++++++++++++++ pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 54 ++++++++ .../k0s.k0sproject.io_clusterconfigs.yaml | 48 +++++++ 4 files changed, 328 insertions(+), 9 deletions(-) diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index f66d3d74d3b4..033ac8a1c8ee 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -37,6 +37,9 @@ type ControlPlaneLoadBalancingSpec struct { // Configuration options related to the VRRP. This is an array which allows // to configure multiple virtual IPs. VRRPInstances VRRPInstances `json:"vrrpInstances,omitempty"` + // Configuration options related to the virtual servers. This is an array + // which allows to configure multiple load balancers. + VirtualServers VirtualServers `json:"virtualServers,omitempty"` } // VRRPInstances is a list of VRRPInstance @@ -85,12 +88,12 @@ func (c *ControlPlaneLoadBalancingSpec) ValidateVRRPInstances(getDefaultNICFn fu if getDefaultNICFn == nil { getDefaultNICFn = getDefaultNIC } - for i, vi := range c.VRRPInstances { - if vi.Name == "" { + for i := range c.VRRPInstances { + if c.VRRPInstances[i].Name == "" { c.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) } - if vi.Interface == "" { + if c.VRRPInstances[i].Interface == "" { nic, err := getDefaultNICFn() if err != nil { return fmt.Errorf("failed to get default NIC: %w", err) @@ -98,29 +101,29 @@ func (c *ControlPlaneLoadBalancingSpec) ValidateVRRPInstances(getDefaultNICFn fu c.VRRPInstances[i].Interface = nic } - if vi.VirtualRouterID == nil { + if c.VRRPInstances[i].VirtualRouterID == nil { vrid := int32(defaultVirtualRouterID + i) c.VRRPInstances[i].VirtualRouterID = &vrid - } else if *vi.VirtualRouterID < 0 || *vi.VirtualRouterID > 255 { + } else if *c.VRRPInstances[i].VirtualRouterID < 0 || *c.VRRPInstances[i].VirtualRouterID > 255 { return errors.New("VirtualRouterID must be in the range of 1-255") } - if vi.AdvertInterval == nil { + if c.VRRPInstances[i].AdvertInterval == nil { advInt := int32(defaultAdvertInterval) c.VRRPInstances[i].AdvertInterval = &advInt } - if vi.AuthPass == "" { + if c.VRRPInstances[i].AuthPass == "" { return errors.New("AuthPass must be defined") } if len(c.VRRPInstances[i].AuthPass) > 8 { return errors.New("AuthPass must be 8 characters or less") } - if len(vi.VirtualIPs) == 0 { + if len(c.VRRPInstances[i].VirtualIPs) == 0 { return errors.New("VirtualIPs must be defined") } - for _, vip := range vi.VirtualIPs { + for _, vip := range c.VRRPInstances[i].VirtualIPs { if _, _, err := net.ParseCIDR(vip); err != nil { return fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip) } @@ -128,3 +131,101 @@ func (c *ControlPlaneLoadBalancingSpec) ValidateVRRPInstances(getDefaultNICFn fu } return nil } + +// VirtualServers is a list of VirtualServer +type VirtualServers []VirtualServer + +// VirtualServer defines the configuration options for a virtual server. +type VirtualServer struct { + // IPAddress is the virtual IP address used by the virtual server. + IPAddress string `json:"ipAddress"` + // DelayLoop is the delay timer for check polling. If not specified, defaults to 0. + DelayLoop int `json:"delayLoop,omitempty"` + // LBAlgo is the load balancing algorithm. If not specified, defaults to rr. + // Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further + // details refer to keepalived documentation. + LBAlgo KeepalivedLBAlgo `json:"lbAlgo,omitempty"` + // LBKind is the load balancing kind. If not specified, defaults to DR. + // Valid values are NAT DR TUN. For further details refer to keepalived documentation. + LBKind KeepalivedLBKind `json:"lbKind,omitempty"` + // PersistenceTimeout specify a timeout value for persistent connections in + // seconds. If not specified, defaults to 360 (6 minutes). + PersistenceTimeout int `json:"persistenceTimeout,omitempty"` +} + +// KeepalivedLBAlgo describes the load balancing algorithm. +// +kubebuilder:validation:Enum=rr;wrr;lc;wlc;lblc;dh;sh;sed;nq +type KeepalivedLBAlgo string + +const ( + RRAlgo KeepalivedLBAlgo = "rr" + WRRAlgo KeepalivedLBAlgo = "wrr" + LCAlgo KeepalivedLBAlgo = "lc" + WLCAlgo KeepalivedLBAlgo = "wlc" + LBLCAlgo KeepalivedLBAlgo = "lblc" + DHAlgo KeepalivedLBAlgo = "dh" + SHAlgo KeepalivedLBAlgo = "sh" + SEDAlgo KeepalivedLBAlgo = "sed" + NQAlgo KeepalivedLBAlgo = "nq" +) + +// KeepalivedLBKind describes the load balancing forwarding method. +// +kubebuilder:validation:Enum=NAT;DR;TUN +type KeepalivedLBKind string + +const ( + NATLBKind KeepalivedLBKind = "NAT" + DRLBKind KeepalivedLBKind = "DR" + TUNLBKind KeepalivedLBKind = "TUN" +) + +type RealServer struct { + // IPAddress is the IP address of the real server. + IPAddress string `json:"ipAddress"` + // Weight is the weight of the real server. If not specified, defaults to 1. + Weight int `json:"weight,omitempty"` +} + +func (c *ControlPlaneLoadBalancingSpec) ValidateVirtualServers() error { + for i := range c.VirtualServers { + if c.VirtualServers[i].IPAddress == "" { + return errors.New("IPAddress must be defined") + } + if net.ParseIP(c.VirtualServers[i].IPAddress) == nil { + return fmt.Errorf("invalid IP address: %s", c.VirtualServers[i].IPAddress) + } + + if c.VirtualServers[i].LBAlgo == "" { + c.VirtualServers[i].LBAlgo = RRAlgo + } else { + switch c.VirtualServers[i].LBAlgo { + case RRAlgo, WRRAlgo, LCAlgo, WLCAlgo, LBLCAlgo, DHAlgo, SHAlgo, SEDAlgo, NQAlgo: + // valid LBAlgo + default: + return fmt.Errorf("invalid LBAlgo: %s ", c.VirtualServers[i].LBAlgo) + } + } + + if c.VirtualServers[i].LBKind == "" { + c.VirtualServers[i].LBKind = DRLBKind + } else { + switch c.VirtualServers[i].LBKind { + case NATLBKind, DRLBKind, TUNLBKind: + // valid LBKind + default: + return fmt.Errorf("invalid LBKind: %s ", c.VirtualServers[i].LBKind) + } + } + + if c.VirtualServers[i].PersistenceTimeout == 0 { + c.VirtualServers[i].PersistenceTimeout = 360 + } else if c.VirtualServers[i].PersistenceTimeout < 0 { + return errors.New("PersistenceTimeout must be a positive integer") + } + + if c.VirtualServers[i].DelayLoop < 0 { + return errors.New("DelayLoop must be a positive integer") + } + } + return nil +} diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index c257ca2ca075..7a8a46cd7f76 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -150,6 +150,122 @@ func returnNIC() (string, error) { return "fake-nic-0", nil } +func (s *CPLBSuite) TestValidateVirtualServers() { + tests := []struct { + name string + vss []VirtualServer + expectedVSS []VirtualServer + wantErr bool + }{ + { + name: "Set expected defaults", + vss: []VirtualServer{ + { + IPAddress: "1.2.3.4", + }, + { + IPAddress: "1.2.3.5", + }, + }, + expectedVSS: []VirtualServer{ + { + IPAddress: "1.2.3.4", + DelayLoop: 0, + LBAlgo: RRAlgo, + LBKind: DRLBKind, + PersistenceTimeout: 360, + }, + { + IPAddress: "1.2.3.5", + DelayLoop: 0, + LBAlgo: RRAlgo, + LBKind: DRLBKind, + PersistenceTimeout: 360, + }, + }, + wantErr: false, + }, + { + name: "valid instance no overrides", + vss: []VirtualServer{ + { + IPAddress: "1.2.3.4", + DelayLoop: 1, + LBAlgo: WRRAlgo, + LBKind: NATLBKind, + PersistenceTimeout: 100, + }, + }, + expectedVSS: []VirtualServer{ + { + IPAddress: "1.2.3.4", + DelayLoop: 1, + LBAlgo: WRRAlgo, + LBKind: NATLBKind, + PersistenceTimeout: 100, + }, + }, + wantErr: false, + }, + { + name: "empty ip address", + vss: []VirtualServer{{}}, + wantErr: true, + }, + { + name: "invalid IP address", + vss: []VirtualServer{{ + IPAddress: "INVALID", + }}, + wantErr: true, + }, + { + name: "invalid LBAlgo", + vss: []VirtualServer{{ + LBAlgo: "invalid", + }}, + wantErr: true, + }, + { + name: "invalid LBKind", + vss: []VirtualServer{{ + LBKind: "invalid", + }}, + wantErr: true, + }, + { + name: "invalid persistencee timeout", + vss: []VirtualServer{{ + PersistenceTimeout: -1, + }}, + wantErr: true, + }, + { + name: "invalid delay loop", + vss: []VirtualServer{{ + DelayLoop: -1, + }}, + wantErr: true, + }, + } + for _, tt := range tests { + s.Run(tt.name, func() { + cplb := &ControlPlaneLoadBalancingSpec{VirtualServers: tt.vss} + err := cplb.ValidateVirtualServers() + if tt.wantErr { + s.Require().Errorf(err, "Test case %s expected error. Got none", tt.name) + } else { + s.Require().NoErrorf(err, "Tedst case %s expected no error. Got: %v", tt.name, err) + for i := range tt.expectedVSS { + s.Require().Equal(tt.expectedVSS[i].DelayLoop, cplb.VirtualServers[i].DelayLoop, "DelayLoop mismatch") + s.Require().Equal(tt.expectedVSS[i].LBAlgo, cplb.VirtualServers[i].LBAlgo, "LBalgo mismatch") + s.Require().Equal(tt.expectedVSS[i].LBKind, cplb.VirtualServers[i].LBKind, "LBKind mismatch") + s.Require().Equal(tt.expectedVSS[i].PersistenceTimeout, cplb.VirtualServers[i].PersistenceTimeout, "PersistenceTimeout mismatch") + } + } + }) + } +} func TestCPLBSuite(t *testing.T) { cplbSuite := &CPLBSuite{} diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index e5ff2ff9a217..7a5779fecf3c 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -395,6 +395,11 @@ func (in *ControlPlaneLoadBalancingSpec) DeepCopyInto(out *ControlPlaneLoadBalan (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.VirtualServers != nil { + in, out := &in.VirtualServers, &out.VirtualServers + *out = make(VirtualServers, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneLoadBalancingSpec. @@ -865,6 +870,21 @@ func (in *NodeLocalLoadBalancing) DeepCopy() *NodeLocalLoadBalancing { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RealServer) DeepCopyInto(out *RealServer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RealServer. +func (in *RealServer) DeepCopy() *RealServer { + if in == nil { + return nil + } + out := new(RealServer) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in RepositoriesSettings) DeepCopyInto(out *RepositoriesSettings) { { @@ -1046,6 +1066,40 @@ func (in VirtualIPs) DeepCopy() VirtualIPs { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualServer) DeepCopyInto(out *VirtualServer) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualServer. +func (in *VirtualServer) DeepCopy() *VirtualServer { + if in == nil { + return nil + } + out := new(VirtualServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in VirtualServers) DeepCopyInto(out *VirtualServers) { + { + in := &in + *out = make(VirtualServers, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualServers. +func (in VirtualServers) DeepCopy() VirtualServers { + if in == nil { + return nil + } + out := new(VirtualServers) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkerProfile) DeepCopyInto(out *WorkerProfile) { *out = *in diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index c8533d90fed1..d598fa65f397 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -376,6 +376,54 @@ spec: Indicates if control plane load balancing should be enabled. Default: false type: boolean + virtualServers: + description: |- + Configuration options related to the virtual servers. This is an array + which allows to configure multiple load balancers. + items: + description: VirtualServer defines the configuration options + for a virtual server. + properties: + delayLoop: + description: DelayLoop is the delay timer for check + polling. If not specified, defaults to 0. + type: integer + ipAddress: + description: IPAddress is the virtual IP address used + by the virtual server. + type: string + lbAlgo: + description: |- + LBAlgo is the load balancing algorithm. If not specified, defaults to rr. + Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further + details refer to keepalived documentation. + enum: + - rr + - wrr + - lc + - wlc + - lblc + - dh + - sh + - sed + - nq + type: string + lbKind: + description: |- + LBKind is the load balancing kind. If not specified, defaults to DR. + Valid values are NAT DR TUN. For further details refer to keepalived documentation. + enum: + - NAT + - DR + - TUN + type: string + persistenceTimeout: + description: |- + PersistenceTimeout specify a timeout value for persistent connections in + seconds. If not specified, defaults to 360 (6 minutes). + type: integer + type: object + type: array vrrpInstances: description: |- Configuration options related to the VRRP. This is an array which allows From c3a0a58b4f5e5d8efa1db76c3cbce297790c606c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 29 Apr 2024 13:54:04 +0200 Subject: [PATCH 02/12] Implement CPLB virtualServers and reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- cmd/controller/controller.go | 2 + pkg/component/controller/cplb_reconciler.go | 138 ++++++++++++++++++ .../controller/cplb_reconciler_test.go | 84 +++++++++++ pkg/component/controller/cplb_unix.go | 137 ++++++++++++++--- pkg/component/controller/cplb_windows.go | 4 +- pkg/supervisor/supervisor.go | 3 + 6 files changed, 345 insertions(+), 23 deletions(-) create mode 100644 pkg/component/controller/cplb_reconciler.go create mode 100644 pkg/component/controller/cplb_reconciler_test.go diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 6c8deda1b805..defaeb5d3886 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -237,6 +237,8 @@ func (c *command) start(ctx context.Context) error { K0sVars: c.K0sVars, Config: cplb, DetailedLogging: c.Debug, + KubeConfigPath: c.K0sVars.AdminKubeConfigPath, + APISpec: nodeConfig.Spec.API, }) } diff --git a/pkg/component/controller/cplb_reconciler.go b/pkg/component/controller/cplb_reconciler.go new file mode 100644 index 000000000000..f6f89feb194d --- /dev/null +++ b/pkg/component/controller/cplb_reconciler.go @@ -0,0 +1,138 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "time" + + kubeutil "github.com/k0sproject/k0s/pkg/kubernetes" + "github.com/k0sproject/k0s/pkg/kubernetes/watch" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +// CPLBReconciler monitors the endpoints of the "kubernetes" service in the +// "default" namespace. It notifies changes though the updateCh channel provided +// in the constructor. +type CPLBReconciler struct { + log *logrus.Entry + kubeconfigPath string + addresses []string + mu sync.RWMutex + updateCh chan<- struct{} + watchCancelFunc context.CancelFunc +} + +func NewCPLBReconciler(kubeconfigPath string, updateCh chan<- struct{}) *CPLBReconciler { + return &CPLBReconciler{ + log: logrus.WithField("component", "cplb-reconciler"), + kubeconfigPath: kubeconfigPath, + updateCh: updateCh, + } +} + +func (r *CPLBReconciler) Start() error { + clientset, err := kubeutil.NewClientFromFile(r.kubeconfigPath) + if err != nil { + return fmt.Errorf("failed to get clientset: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + r.watchCancelFunc = cancel + go r.watchAPIServers(ctx, clientset) + + return nil +} + +func (r *CPLBReconciler) Stop() { + r.log.Info("Stopping CPLB reconciler") + r.watchCancelFunc() +} + +func (r *CPLBReconciler) watchAPIServers(ctx context.Context, clientSet kubernetes.Interface) { + for { + select { + default: + err := watch.Endpoints(clientSet.CoreV1().Endpoints("default")). + WithObjectName("kubernetes"). + Until(ctx, func(endpoints *corev1.Endpoints) (bool, error) { + r.maybeUpdateIPs(endpoints) + return false, nil + }) + // Log any reconciliation errors, but only if they don't + // indicate that the reconciler has been stopped. + if err != nil && !errors.Is(err, ctx.Err()) { + r.log.WithError(err).Error("Failed to reconcile API server addresses") + } + + // After a watch error wait 5 seconds before retrying + time.Sleep(5 * time.Second) + + case <-ctx.Done(): + r.log.Info("Stopped watching kubernetes endpoints") + return + } + } +} + +// maybeUpdateIPs updates the list of IP addresses if the new list has +// different addresses +func (r *CPLBReconciler) maybeUpdateIPs(endpoint *corev1.Endpoints) { + newAddresses := []string{} + for _, subset := range endpoint.Subsets { + for _, addr := range subset.Addresses { + newAddresses = append(newAddresses, addr.IP) + } + } + + r.mu.Lock() + defer r.mu.Unlock() + + // endpoints are not guaranteed to be sorted by IP address + slices.Sort(newAddresses) + + if !slices.Equal(r.addresses, newAddresses) { + r.addresses = newAddresses + r.log.Infof("Updated the list of IPs: %v", r.addresses) + select { + case r.updateCh <- struct{}{}: + default: + } + } +} + +func (r *CPLBReconciler) updateAddresses(newAddresses []string) { + r.addresses = newAddresses + r.log.Infof("Updated the list of IPs: %v", r.addresses) + select { + case r.updateCh <- struct{}{}: + default: + } +} + +// GetIPs gets a thread-safe copy of the current list of IP addresses +func (r *CPLBReconciler) GetIPs() []string { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.addresses) +} diff --git a/pkg/component/controller/cplb_reconciler_test.go b/pkg/component/controller/cplb_reconciler_test.go new file mode 100644 index 000000000000..029661346834 --- /dev/null +++ b/pkg/component/controller/cplb_reconciler_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" +) + +type CPLBReconcilerSuite struct { + suite.Suite +} + +func (s *CPLBReconcilerSuite) TestMaybeUpdateIPs() { + ch := make(chan struct{}, 1) + var updateCh <-chan struct{} = ch + reconciler := &CPLBReconciler{ + addresses: []string{}, + updateCh: ch, + log: logrus.WithField("component", "cplb-reconciler-test"), + } + + endpoints := &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + {IP: "192.168.1.1"}, + {IP: "192.168.1.2"}, + }, + }, + }, + } + + // test the addresses change when the endpoints change and the channel is notified + reconciler.maybeUpdateIPs(endpoints) + select { + case <-updateCh: + s.Require().Equal([]string{"192.168.1.1", "192.168.1.2"}, reconciler.GetIPs(), "Expected the addresses to be updated") + default: + s.FailNow("Expected an update signal on the updateCh channel") + } + + // test the addresses don't change when the endpoints don't change and the channel isn't notified. + reconciler.maybeUpdateIPs(endpoints) + select { + case <-updateCh: + s.FailNow("Expected no update signal on the updateCh channel") + default: + s.Require().Equal([]string{"192.168.1.1", "192.168.1.2"}, reconciler.GetIPs(), "Unexpected addresses change") + } + + // test the addresses change when the endpoints change and the channel is notified when the addresses are empty + endpoints.Subsets[0].Addresses = []corev1.EndpointAddress{} + reconciler.maybeUpdateIPs(endpoints) + select { + case <-updateCh: + s.Require().Equal([]string{}, reconciler.GetIPs(), "Expected the addresses to be updated") + default: + s.FailNow("Expected an update signal on the updateCh channel") + } +} + +func TestCPLBReconcilerSuite(t *testing.T) { + cplbReconcilerSuite := &CPLBReconcilerSuite{} + + suite.Run(t, cplbReconcilerSuite) +} diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index 1781894c6abd..211d54e07e4b 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -28,7 +28,9 @@ import ( "os" "path/filepath" "slices" + "syscall" "text/template" + "time" "github.com/k0sproject/k0s/internal/pkg/file" "github.com/k0sproject/k0s/internal/pkg/users" @@ -43,13 +45,18 @@ import ( // Keepalived is the controller for the keepalived process in the control plane load balancing type Keepalived struct { - K0sVars *config.CfgVars - Config *k0sAPI.ControlPlaneLoadBalancingSpec - DetailedLogging bool - uid int - supervisor *supervisor.Supervisor - log *logrus.Entry - configFilePath string + K0sVars *config.CfgVars + Config *k0sAPI.ControlPlaneLoadBalancingSpec + DetailedLogging bool + APISpec *k0sAPI.APISpec + KubeConfigPath string + keepalivedConfig *keepalivedConfig + uid int + supervisor *supervisor.Supervisor + log *logrus.Entry + configFilePath string + reconciler *CPLBReconciler + updateCh <-chan struct{} } // Init extracts the needed binaries and creates the directories @@ -71,20 +78,45 @@ func (k *Keepalived) Init(_ context.Context) error { // Start generates the keepalived configuration and starts the keepalived process func (k *Keepalived) Start(_ context.Context) error { - if k.Config == nil { + if k.Config == nil || (len(k.Config.VRRPInstances) == 0 && len(k.Config.VirtualServers) == 0) { + k.log.Warn("No VRRP instances or virtual servers defined, skipping keepalived start") return nil } - if err := k.configureDummy(); err != nil { - return fmt.Errorf("failed to configure dummy interface: %w", err) + if len(k.Config.VRRPInstances) > 0 { + if err := k.configureDummy(); err != nil { + return fmt.Errorf("failed to configure dummy interface: %w", err) + } + if err := k.Config.ValidateVRRPInstances(nil); err != nil { + return fmt.Errorf("failed to validate VRRP instances: %w", err) + } + } + + if len(k.Config.VirtualServers) > 0 { + if k.APISpec.ExternalAddress != "" { + return errors.New("externalAddress is not supported with virtual servers") + } + if err := k.Config.ValidateVirtualServers(); err != nil { + return fmt.Errorf("failed to validate virtual servers: %w", err) + } + k.log.Info("Starting CPLB reconciler") + updateCh := make(chan struct{}, 1) + k.reconciler = NewCPLBReconciler(k.KubeConfigPath, updateCh) + k.updateCh = updateCh + if err := k.reconciler.Start(); err != nil { + return fmt.Errorf("failed to start CPLB reconciler: %w", err) + } } - if err := k.Config.ValidateVRRPInstances(nil); err != nil { - return fmt.Errorf("failed to validate VRRP instances: %w", err) + // In order to make the code simpler, we always create the keepalived template + // without the virtual servers, before starting the reconcile loop + k.keepalivedConfig = &keepalivedConfig{ + VRRPInstances: k.Config.VRRPInstances, + VirtualServers: k.Config.VirtualServers, + APIServerPort: k.APISpec.Port, } if err := k.generateKeepalivedTemplate(); err != nil { return fmt.Errorf("failed to generate keepalived template: %w", err) - } args := []string{ @@ -108,6 +140,10 @@ func (k *Keepalived) Start(_ context.Context) error { DataDir: k.K0sVars.DataDir, UID: k.uid, } + + if len(k.Config.VirtualServers) > 0 { + go k.watchReconcilerUpdates() + } return k.supervisor.Supervise() } @@ -120,6 +156,11 @@ func (k *Keepalived) Stop() error { return fmt.Errorf("failed to stop keepalived: %w", err) } + k.log.Infof("Stopping cplb-reconciler") + if k.reconciler != nil { + k.reconciler.Stop() + } + k.log.Infof("Deleting dummy interface") link, err := netlink.LinkByName(dummyLinkName) if err != nil { @@ -268,18 +309,13 @@ func (*Keepalived) getLinkAddresses(link netlink.Link) ([]netlink.Addr, []string } func (k *Keepalived) generateKeepalivedTemplate() error { - template := template.Must(template.New("keepalived").Parse(keepalivedConfigTemplate)) - kc := keepalivedConfig{ - VRRPInstances: k.Config.VRRPInstances, - } - if err := file.WriteAtomically(k.configFilePath, 0400, func(file io.Writer) error { if err := file.(*os.File).Chown(k.uid, -1); err != nil { return err } w := bufio.NewWriter(file) - if err := template.Execute(w, kc); err != nil { + if err := keepalivedConfigTemplate.Execute(w, k.keepalivedConfig); err != nil { return err } return w.Flush() @@ -290,16 +326,47 @@ func (k *Keepalived) generateKeepalivedTemplate() error { return nil } +func (k *Keepalived) watchReconcilerUpdates() { + // Wait for the supervisor to start keepalived before + // watching for endpoint changes + process := k.supervisor.GetProcess() + for process == nil { + k.log.Info("Waiting for keepalived to start") + time.Sleep(5 * time.Second) + process = k.supervisor.GetProcess() + } + + k.log.Info("started watching cplb-reconciler updates") + for range k.updateCh { + k.keepalivedConfig.RealServers = k.reconciler.GetIPs() + k.log.Infof("cplb-reconciler update, got %s", k.keepalivedConfig.RealServers) + if err := k.generateKeepalivedTemplate(); err != nil { + k.log.Errorf("failed to generate keepalived template: %v", err) + continue + } + + process := k.supervisor.GetProcess() + if err := process.Signal(syscall.SIGHUP); err != nil { + k.log.Errorf("failed to send SIGHUP to keepalived: %v", err) + } + } + k.log.Info("stopped watching cplb-reconciler updates") +} + // keepalivedConfig contains all the information required by the // KeepalivedConfigTemplate. // Right now this struct doesn't make sense right now but we need this for the // future virtual_server support. type keepalivedConfig struct { - VRRPInstances []k0sAPI.VRRPInstance + VRRPInstances []k0sAPI.VRRPInstance + VirtualServers []k0sAPI.VirtualServer + RealServers []string + APIServerPort int } const dummyLinkName = "dummyvip0" -const keepalivedConfigTemplate = ` + +var keepalivedConfigTemplate = template.Must(template.New("keepalived").Parse(` {{ range .VRRPInstances }} vrrp_instance {{ .Name }} { # All servers must have state BACKUP so that when a new server comes up @@ -325,4 +392,30 @@ vrrp_instance {{ .Name }} { } } {{ end }} -` + +{{ $APIServerPort := .APIServerPort }} +{{ $RealServers := .RealServers }} +{{ if gt (len $RealServers) 0 }} +{{ range .VirtualServers }} +virtual_server {{ .IPAddress }} {{ $APIServerPort }} { + delay_loop {{ .DelayLoop }} + lb_algo {{ .LBAlgo }} + lb_kind {{ .LBKind }} + persistence_timeout {{ .PersistenceTimeoutSeconds }} + protocol TCP + + {{ range $RealServers }} + real_server {{ . }} {{ $APIServerPort }} { + weight 1 + TCP_CHECK { + warmup 0 + retry 1 + connect_timeout 3 + connect_port {{ $APIServerPort }} + } + } + {{end}} +} +{{ end }} +{{ end }} +`)) diff --git a/pkg/component/controller/cplb_windows.go b/pkg/component/controller/cplb_windows.go index c6eeef5889fe..e96a93108f76 100644 --- a/pkg/component/controller/cplb_windows.go +++ b/pkg/component/controller/cplb_windows.go @@ -28,8 +28,10 @@ import ( // Just create the interface so that the CI doesn't complain. type Keepalived struct { K0sVars *config.CfgVars - Config *k0sAPI.ControlPlaneLoadBalancingSpec + Config *k0sAPI.KeepalivedSpec DetailedLogging bool + APISpec *k0sAPI.APISpec + KubeConfigPath string } func (k *Keepalived) Init(_ context.Context) error { diff --git a/pkg/supervisor/supervisor.go b/pkg/supervisor/supervisor.go index 07165a7f2c40..4b3d2f968d49 100644 --- a/pkg/supervisor/supervisor.go +++ b/pkg/supervisor/supervisor.go @@ -302,5 +302,8 @@ func getEnv(dataDir, component string, keepEnvPrefix bool) []string { func (s *Supervisor) GetProcess() *os.Process { s.mutex.Lock() defer s.mutex.Unlock() + if s.cmd == nil { + return nil + } return s.cmd.Process } From 661ef19d65457bb06ea86ae52b5dd60fdee00d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 6 May 2024 16:40:20 +0200 Subject: [PATCH 03/12] Extract keepalived configuration to a subtype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And move validation to clusterconfig. Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- cmd/controller/controller.go | 4 +- inttest/cplb/cplb_test.go | 7 +- pkg/apis/k0s/v1beta1/clusterconfig_types.go | 6 + pkg/apis/k0s/v1beta1/cplb.go | 153 ++++++++---- pkg/apis/k0s/v1beta1/cplb_test.go | 79 ++++--- pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 42 +++- pkg/component/controller/cplb_reconciler.go | 9 - pkg/component/controller/cplb_unix.go | 15 +- pkg/component/controller/cplb_windows.go | 2 +- .../k0s.k0sproject.io_clusterconfigs.yaml | 220 ++++++++++-------- 10 files changed, 315 insertions(+), 222 deletions(-) diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index defaeb5d3886..412fe356e80c 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -235,10 +235,10 @@ func (c *command) start(ctx context.Context) error { nodeComponents.Add(ctx, &controller.Keepalived{ K0sVars: c.K0sVars, - Config: cplb, + Config: cplb.Keepalived, DetailedLogging: c.Debug, KubeConfigPath: c.K0sVars.AdminKubeConfigPath, - APISpec: nodeConfig.Spec.API, + APIPort: nodeConfig.Spec.API.Port, }) } diff --git a/inttest/cplb/cplb_test.go b/inttest/cplb/cplb_test.go index 4280050d110e..6a848b597ec9 100644 --- a/inttest/cplb/cplb_test.go +++ b/inttest/cplb/cplb_test.go @@ -38,9 +38,10 @@ spec: network: controlPlaneLoadBalancing: enabled: true - vrrpInstances: - - virtualIPs: ["%s/24"] - authPass: "123456" + type: Keepalived + vrrpInstances: + - virtualIPs: ["%s/24"] + authPass: "123456" ` // SetupTest prepares the controller and filesystem, getting it into a consistent diff --git a/pkg/apis/k0s/v1beta1/clusterconfig_types.go b/pkg/apis/k0s/v1beta1/clusterconfig_types.go index 7dfd99da5aa3..cdbebe68487c 100644 --- a/pkg/apis/k0s/v1beta1/clusterconfig_types.go +++ b/pkg/apis/k0s/v1beta1/clusterconfig_types.go @@ -335,6 +335,12 @@ func (s *ClusterSpec) Validate() (errs []error) { errs = append(errs, err) } + if s.Network != nil && s.Network.ControlPlaneLoadBalancing != nil { + for _, err := range s.Network.ControlPlaneLoadBalancing.Validate(s.API.ExternalAddress) { + errs = append(errs, fmt.Errorf("controlPlaneLoadBalancing: %w", err)) + } + } + return } diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index 033ac8a1c8ee..ee55a3ce316e 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -34,6 +34,28 @@ type ControlPlaneLoadBalancingSpec struct { // +optional Enabled bool `json:"enabled,omitempty"` + // type indicates the type of the node-local load balancer to deploy on + // worker nodes. Currently, the only supported type is "Keepalived". + // +kubebuilder:default=Keepalived + // +optional + Type CPLBType `json:"type,omitempty"` + + // Keepalived contains configuration options related to the "Keepalived" type + // of load balancing. + Keepalived *KeepalivedSpec `json:"keepalived,omitempty"` +} + +// NllbType describes which type of load balancer should be deployed for the +// node-local load balancing. The default is [CPLBTypeKeepalived]. +// +kubebuilder:validation:Enum=Keepalived +type CPLBType string + +const ( + // CPLBTypeKeepalived selects Keepalived as the backing load balancer. + CPLBTypeKeepalived CPLBType = "Keepalived" +) + +type KeepalivedSpec struct { // Configuration options related to the VRRP. This is an array which allows // to configure multiple virtual IPs. VRRPInstances VRRPInstances `json:"vrrpInstances,omitempty"` @@ -82,75 +104,86 @@ type VRRPInstance struct { type VirtualIPs []string -// ValidateVRRPInstances validates existing configuration and sets the default +// validateVRRPInstances validates existing configuration and sets the default // values of undefined fields. -func (c *ControlPlaneLoadBalancingSpec) ValidateVRRPInstances(getDefaultNICFn func() (string, error)) error { +func (k *KeepalivedSpec) validateVRRPInstances(getDefaultNICFn func() (string, error)) []error { + errs := []error{} if getDefaultNICFn == nil { getDefaultNICFn = getDefaultNIC } - for i := range c.VRRPInstances { - if c.VRRPInstances[i].Name == "" { - c.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) + for i := range k.VRRPInstances { + if k.VRRPInstances[i].Name == "" { + k.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) } - if c.VRRPInstances[i].Interface == "" { + if k.VRRPInstances[i].Interface == "" { nic, err := getDefaultNICFn() if err != nil { - return fmt.Errorf("failed to get default NIC: %w", err) + errs = append(errs, fmt.Errorf("failed to get default NIC: %w", err)) } - c.VRRPInstances[i].Interface = nic + k.VRRPInstances[i].Interface = nic } - if c.VRRPInstances[i].VirtualRouterID == nil { + if k.VRRPInstances[i].VirtualRouterID == nil { vrid := int32(defaultVirtualRouterID + i) - c.VRRPInstances[i].VirtualRouterID = &vrid - } else if *c.VRRPInstances[i].VirtualRouterID < 0 || *c.VRRPInstances[i].VirtualRouterID > 255 { - return errors.New("VirtualRouterID must be in the range of 1-255") + k.VRRPInstances[i].VirtualRouterID = &vrid + } else if *k.VRRPInstances[i].VirtualRouterID < 0 || *k.VRRPInstances[i].VirtualRouterID > 255 { + errs = append(errs, errors.New("VirtualRouterID must be in the range of 1-255")) } - if c.VRRPInstances[i].AdvertInterval == nil { + if k.VRRPInstances[i].AdvertInterval == nil { advInt := int32(defaultAdvertInterval) - c.VRRPInstances[i].AdvertInterval = &advInt + k.VRRPInstances[i].AdvertInterval = &advInt } - if c.VRRPInstances[i].AuthPass == "" { - return errors.New("AuthPass must be defined") + if k.VRRPInstances[i].AuthPass == "" { + errs = append(errs, errors.New("AuthPass must be defined")) } - if len(c.VRRPInstances[i].AuthPass) > 8 { - return errors.New("AuthPass must be 8 characters or less") + if len(k.VRRPInstances[i].AuthPass) > 8 { + errs = append(errs, errors.New("AuthPass must be 8 characters or less")) } - if len(c.VRRPInstances[i].VirtualIPs) == 0 { - return errors.New("VirtualIPs must be defined") + if len(k.VRRPInstances[i].VirtualIPs) == 0 { + errs = append(errs, errors.New("VirtualIPs must be defined")) } - for _, vip := range c.VRRPInstances[i].VirtualIPs { + for _, vip := range k.VRRPInstances[i].VirtualIPs { if _, _, err := net.ParseCIDR(vip); err != nil { - return fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip) + errs = append(errs, fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip)) } } } - return nil + return errs } // VirtualServers is a list of VirtualServer +// +listType=map +// +listMapKey=ipAddress type VirtualServers []VirtualServer // VirtualServer defines the configuration options for a virtual server. type VirtualServer struct { // IPAddress is the virtual IP address used by the virtual server. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 IPAddress string `json:"ipAddress"` // DelayLoop is the delay timer for check polling. If not specified, defaults to 0. + // kubebuilder:validation:Minimum=0 DelayLoop int `json:"delayLoop,omitempty"` // LBAlgo is the load balancing algorithm. If not specified, defaults to rr. // Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further // details refer to keepalived documentation. + // +kubebuilder:default=rr + // +optional LBAlgo KeepalivedLBAlgo `json:"lbAlgo,omitempty"` // LBKind is the load balancing kind. If not specified, defaults to DR. // Valid values are NAT DR TUN. For further details refer to keepalived documentation. + // +kubebuilder:default=DR + // +optional LBKind KeepalivedLBKind `json:"lbKind,omitempty"` - // PersistenceTimeout specify a timeout value for persistent connections in + // PersistenceTimeoutSeconds specify a timeout value for persistent connections in // seconds. If not specified, defaults to 360 (6 minutes). - PersistenceTimeout int `json:"persistenceTimeout,omitempty"` + // kubebuilder:validation:Minimum=0 + PersistenceTimeoutSeconds int `json:"persistenceTimeoutSeconds,omitempty"` } // KeepalivedLBAlgo describes the load balancing algorithm. @@ -186,46 +219,74 @@ type RealServer struct { Weight int `json:"weight,omitempty"` } -func (c *ControlPlaneLoadBalancingSpec) ValidateVirtualServers() error { - for i := range c.VirtualServers { - if c.VirtualServers[i].IPAddress == "" { - return errors.New("IPAddress must be defined") +// validateVRRPInstances validates existing configuration and sets the default +// values of undefined fields. +func (k *KeepalivedSpec) validateVirtualServers() []error { + errs := []error{} + for i := range k.VirtualServers { + if k.VirtualServers[i].IPAddress == "" { + errs = append(errs, errors.New("IPAddress must be defined")) } - if net.ParseIP(c.VirtualServers[i].IPAddress) == nil { - return fmt.Errorf("invalid IP address: %s", c.VirtualServers[i].IPAddress) + if net.ParseIP(k.VirtualServers[i].IPAddress) == nil { + errs = append(errs, fmt.Errorf("invalid IP address: %s", k.VirtualServers[i].IPAddress)) } - if c.VirtualServers[i].LBAlgo == "" { - c.VirtualServers[i].LBAlgo = RRAlgo + if k.VirtualServers[i].LBAlgo == "" { + k.VirtualServers[i].LBAlgo = RRAlgo } else { - switch c.VirtualServers[i].LBAlgo { + switch k.VirtualServers[i].LBAlgo { case RRAlgo, WRRAlgo, LCAlgo, WLCAlgo, LBLCAlgo, DHAlgo, SHAlgo, SEDAlgo, NQAlgo: // valid LBAlgo default: - return fmt.Errorf("invalid LBAlgo: %s ", c.VirtualServers[i].LBAlgo) + errs = append(errs, fmt.Errorf("invalid LBAlgo: %s ", k.VirtualServers[i].LBAlgo)) } } - if c.VirtualServers[i].LBKind == "" { - c.VirtualServers[i].LBKind = DRLBKind + if k.VirtualServers[i].LBKind == "" { + k.VirtualServers[i].LBKind = DRLBKind } else { - switch c.VirtualServers[i].LBKind { + switch k.VirtualServers[i].LBKind { case NATLBKind, DRLBKind, TUNLBKind: // valid LBKind default: - return fmt.Errorf("invalid LBKind: %s ", c.VirtualServers[i].LBKind) + errs = append(errs, fmt.Errorf("invalid LBKind: %s ", k.VirtualServers[i].LBKind)) } } - if c.VirtualServers[i].PersistenceTimeout == 0 { - c.VirtualServers[i].PersistenceTimeout = 360 - } else if c.VirtualServers[i].PersistenceTimeout < 0 { - return errors.New("PersistenceTimeout must be a positive integer") + if k.VirtualServers[i].PersistenceTimeoutSeconds == 0 { + k.VirtualServers[i].PersistenceTimeoutSeconds = 360 + } else if k.VirtualServers[i].PersistenceTimeoutSeconds < 0 { + errs = append(errs, errors.New("PersistenceTimeout must be a positive integer")) } - if c.VirtualServers[i].DelayLoop < 0 { - return errors.New("DelayLoop must be a positive integer") + if k.VirtualServers[i].DelayLoop < 0 { + errs = append(errs, errors.New("DelayLoop must be a positive integer")) } } - return nil + return errs +} + +// Validate validates the ControlPlaneLoadBalancingSpec +func (c *ControlPlaneLoadBalancingSpec) Validate(externalAddress string) []error { + if c == nil { + return nil + } + errs := []error{} + + switch c.Type { + case CPLBTypeKeepalived: + case "": + c.Type = CPLBTypeKeepalived + default: + errs = append(errs, fmt.Errorf("unsupported CPLB type: %s. Only allowed value: %s", c.Type, CPLBTypeKeepalived)) + } + + errs = append(errs, c.Keepalived.validateVRRPInstances(nil)...) + errs = append(errs, c.Keepalived.validateVirtualServers()...) + // CPLB reconciler relies in watching kubernetes.default.svc endpoints + if externalAddress != "" && len(c.Keepalived.VirtualServers) > 0 { + errs = append(errs, errors.New(".spec.api.externalAddress and VRRPInstances cannot be used together")) + } + + return errs } diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index 7a8a46cd7f76..c42c6d28161a 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -28,7 +28,6 @@ type CPLBSuite struct { } func (s *CPLBSuite) TestValidateVRRPInstances() { - tests := []struct { name string vrrps []VRRPInstance @@ -125,21 +124,21 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { for _, tt := range tests { s.Run(tt.name, func() { - elb := &ControlPlaneLoadBalancingSpec{ + k := &KeepalivedSpec{ VRRPInstances: tt.vrrps, } - err := elb.ValidateVRRPInstances(returnNIC) + err := k.validateVRRPInstances(returnNIC) if tt.wantErr { - s.Require().Errorf(err, "Test case %s expected error. Got none", tt.name) + s.Require().NotEmpty(err, "Test case %s expected error. Got none", tt.name) } else { - s.Require().NoErrorf(err, "Test case %s expected no error. Got: %v", tt.name, err) - s.T().Log(elb.VRRPInstances) - s.Require().Equal(len(tt.expectedVRRPs), len(elb.VRRPInstances), "Expected and actual VRRPInstances length mismatch") + s.Require().Empty(err, "Test case %s expected no errors. Got: %v", tt.name, err) + s.T().Log(k.VRRPInstances) + s.Require().Equal(len(tt.expectedVRRPs), len(k.VRRPInstances), "Expected and actual VRRPInstances length mismatch") for i := 0; i < len(tt.expectedVRRPs); i++ { - s.Require().Equal(tt.expectedVRRPs[i].Name, elb.VRRPInstances[i].Name, "Name mismatch") - s.Require().Equal(tt.expectedVRRPs[i].Interface, elb.VRRPInstances[i].Interface, "Interface mismatch") - s.Require().Equal(*tt.expectedVRRPs[i].VirtualRouterID, *elb.VRRPInstances[i].VirtualRouterID, "Virtual router ID mismatch") - s.Require().Equal(*tt.expectedVRRPs[i].AdvertInterval, *elb.VRRPInstances[i].AdvertInterval, "Virtual router ID mismatch") + s.Require().Equal(tt.expectedVRRPs[i].Name, k.VRRPInstances[i].Name, "Name mismatch") + s.Require().Equal(tt.expectedVRRPs[i].Interface, k.VRRPInstances[i].Interface, "Interface mismatch") + s.Require().Equal(*tt.expectedVRRPs[i].VirtualRouterID, *k.VRRPInstances[i].VirtualRouterID, "Virtual router ID mismatch") + s.Require().Equal(*tt.expectedVRRPs[i].AdvertInterval, *k.VRRPInstances[i].AdvertInterval, "Virtual router ID mismatch") } } }) @@ -169,18 +168,18 @@ func (s *CPLBSuite) TestValidateVirtualServers() { }, expectedVSS: []VirtualServer{ { - IPAddress: "1.2.3.4", - DelayLoop: 0, - LBAlgo: RRAlgo, - LBKind: DRLBKind, - PersistenceTimeout: 360, + IPAddress: "1.2.3.4", + DelayLoop: 0, + LBAlgo: RRAlgo, + LBKind: DRLBKind, + PersistenceTimeoutSeconds: 360, }, { - IPAddress: "1.2.3.5", - DelayLoop: 0, - LBAlgo: RRAlgo, - LBKind: DRLBKind, - PersistenceTimeout: 360, + IPAddress: "1.2.3.5", + DelayLoop: 0, + LBAlgo: RRAlgo, + LBKind: DRLBKind, + PersistenceTimeoutSeconds: 360, }, }, wantErr: false, @@ -189,20 +188,20 @@ func (s *CPLBSuite) TestValidateVirtualServers() { name: "valid instance no overrides", vss: []VirtualServer{ { - IPAddress: "1.2.3.4", - DelayLoop: 1, - LBAlgo: WRRAlgo, - LBKind: NATLBKind, - PersistenceTimeout: 100, + IPAddress: "1.2.3.4", + DelayLoop: 1, + LBAlgo: WRRAlgo, + LBKind: NATLBKind, + PersistenceTimeoutSeconds: 100, }, }, expectedVSS: []VirtualServer{ { - IPAddress: "1.2.3.4", - DelayLoop: 1, - LBAlgo: WRRAlgo, - LBKind: NATLBKind, - PersistenceTimeout: 100, + IPAddress: "1.2.3.4", + DelayLoop: 1, + LBAlgo: WRRAlgo, + LBKind: NATLBKind, + PersistenceTimeoutSeconds: 100, }, }, wantErr: false, @@ -236,7 +235,7 @@ func (s *CPLBSuite) TestValidateVirtualServers() { { name: "invalid persistencee timeout", vss: []VirtualServer{{ - PersistenceTimeout: -1, + PersistenceTimeoutSeconds: -1, }}, wantErr: true, }, @@ -250,17 +249,17 @@ func (s *CPLBSuite) TestValidateVirtualServers() { } for _, tt := range tests { s.Run(tt.name, func() { - cplb := &ControlPlaneLoadBalancingSpec{VirtualServers: tt.vss} - err := cplb.ValidateVirtualServers() + k := &KeepalivedSpec{VirtualServers: tt.vss} + err := k.validateVirtualServers() if tt.wantErr { - s.Require().Errorf(err, "Test case %s expected error. Got none", tt.name) + s.Require().NotEmpty(err, "Test case %s expected error. Got none", tt.name) } else { - s.Require().NoErrorf(err, "Tedst case %s expected no error. Got: %v", tt.name, err) + s.Require().Empty(err, "Tedst case %s expected no error. Got: %v", tt.name, err) for i := range tt.expectedVSS { - s.Require().Equal(tt.expectedVSS[i].DelayLoop, cplb.VirtualServers[i].DelayLoop, "DelayLoop mismatch") - s.Require().Equal(tt.expectedVSS[i].LBAlgo, cplb.VirtualServers[i].LBAlgo, "LBalgo mismatch") - s.Require().Equal(tt.expectedVSS[i].LBKind, cplb.VirtualServers[i].LBKind, "LBKind mismatch") - s.Require().Equal(tt.expectedVSS[i].PersistenceTimeout, cplb.VirtualServers[i].PersistenceTimeout, "PersistenceTimeout mismatch") + s.Require().Equal(tt.expectedVSS[i].DelayLoop, k.VirtualServers[i].DelayLoop, "DelayLoop mismatch") + s.Require().Equal(tt.expectedVSS[i].LBAlgo, k.VirtualServers[i].LBAlgo, "LBalgo mismatch") + s.Require().Equal(tt.expectedVSS[i].LBKind, k.VirtualServers[i].LBKind, "LBKind mismatch") + s.Require().Equal(tt.expectedVSS[i].PersistenceTimeoutSeconds, k.VirtualServers[i].PersistenceTimeoutSeconds, "PersistenceTimeout mismatch") } } }) diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index 7a5779fecf3c..cb2a69b71d16 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -388,17 +388,10 @@ func (in *ClusterTelemetry) DeepCopy() *ClusterTelemetry { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlaneLoadBalancingSpec) DeepCopyInto(out *ControlPlaneLoadBalancingSpec) { *out = *in - if in.VRRPInstances != nil { - in, out := &in.VRRPInstances, &out.VRRPInstances - *out = make(VRRPInstances, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.VirtualServers != nil { - in, out := &in.VirtualServers, &out.VirtualServers - *out = make(VirtualServers, len(*in)) - copy(*out, *in) + if in.Keepalived != nil { + in, out := &in.Keepalived, &out.Keepalived + *out = new(KeepalivedSpec) + (*in).DeepCopyInto(*out) } } @@ -658,6 +651,33 @@ func (in *InstallSpec) DeepCopy() *InstallSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeepalivedSpec) DeepCopyInto(out *KeepalivedSpec) { + *out = *in + if in.VRRPInstances != nil { + in, out := &in.VRRPInstances, &out.VRRPInstances + *out = make(VRRPInstances, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VirtualServers != nil { + in, out := &in.VirtualServers, &out.VirtualServers + *out = make(VirtualServers, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeepalivedSpec. +func (in *KeepalivedSpec) DeepCopy() *KeepalivedSpec { + if in == nil { + return nil + } + out := new(KeepalivedSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KineConfig) DeepCopyInto(out *KineConfig) { *out = *in diff --git a/pkg/component/controller/cplb_reconciler.go b/pkg/component/controller/cplb_reconciler.go index f6f89feb194d..5deaefb62b67 100644 --- a/pkg/component/controller/cplb_reconciler.go +++ b/pkg/component/controller/cplb_reconciler.go @@ -121,15 +121,6 @@ func (r *CPLBReconciler) maybeUpdateIPs(endpoint *corev1.Endpoints) { } } -func (r *CPLBReconciler) updateAddresses(newAddresses []string) { - r.addresses = newAddresses - r.log.Infof("Updated the list of IPs: %v", r.addresses) - select { - case r.updateCh <- struct{}{}: - default: - } -} - // GetIPs gets a thread-safe copy of the current list of IP addresses func (r *CPLBReconciler) GetIPs() []string { r.mu.Lock() diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index 211d54e07e4b..2075499af1d0 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -46,9 +46,9 @@ import ( // Keepalived is the controller for the keepalived process in the control plane load balancing type Keepalived struct { K0sVars *config.CfgVars - Config *k0sAPI.ControlPlaneLoadBalancingSpec + Config *k0sAPI.KeepalivedSpec DetailedLogging bool - APISpec *k0sAPI.APISpec + APIPort int KubeConfigPath string keepalivedConfig *keepalivedConfig uid int @@ -87,18 +87,9 @@ func (k *Keepalived) Start(_ context.Context) error { if err := k.configureDummy(); err != nil { return fmt.Errorf("failed to configure dummy interface: %w", err) } - if err := k.Config.ValidateVRRPInstances(nil); err != nil { - return fmt.Errorf("failed to validate VRRP instances: %w", err) - } } if len(k.Config.VirtualServers) > 0 { - if k.APISpec.ExternalAddress != "" { - return errors.New("externalAddress is not supported with virtual servers") - } - if err := k.Config.ValidateVirtualServers(); err != nil { - return fmt.Errorf("failed to validate virtual servers: %w", err) - } k.log.Info("Starting CPLB reconciler") updateCh := make(chan struct{}, 1) k.reconciler = NewCPLBReconciler(k.KubeConfigPath, updateCh) @@ -113,7 +104,7 @@ func (k *Keepalived) Start(_ context.Context) error { k.keepalivedConfig = &keepalivedConfig{ VRRPInstances: k.Config.VRRPInstances, VirtualServers: k.Config.VirtualServers, - APIServerPort: k.APISpec.Port, + APIServerPort: k.APIPort, } if err := k.generateKeepalivedTemplate(); err != nil { return fmt.Errorf("failed to generate keepalived template: %w", err) diff --git a/pkg/component/controller/cplb_windows.go b/pkg/component/controller/cplb_windows.go index e96a93108f76..9fd7dae0c429 100644 --- a/pkg/component/controller/cplb_windows.go +++ b/pkg/component/controller/cplb_windows.go @@ -30,7 +30,7 @@ type Keepalived struct { K0sVars *config.CfgVars Config *k0sAPI.KeepalivedSpec DetailedLogging bool - APISpec *k0sAPI.APISpec + APIPort int KubeConfigPath string } diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index d598fa65f397..85d0627bcc3d 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -376,106 +376,130 @@ spec: Indicates if control plane load balancing should be enabled. Default: false type: boolean - virtualServers: + keepalived: description: |- - Configuration options related to the virtual servers. This is an array - which allows to configure multiple load balancers. - items: - description: VirtualServer defines the configuration options - for a virtual server. - properties: - delayLoop: - description: DelayLoop is the delay timer for check - polling. If not specified, defaults to 0. - type: integer - ipAddress: - description: IPAddress is the virtual IP address used - by the virtual server. - type: string - lbAlgo: - description: |- - LBAlgo is the load balancing algorithm. If not specified, defaults to rr. - Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further - details refer to keepalived documentation. - enum: - - rr - - wrr - - lc - - wlc - - lblc - - dh - - sh - - sed - - nq - type: string - lbKind: - description: |- - LBKind is the load balancing kind. If not specified, defaults to DR. - Valid values are NAT DR TUN. For further details refer to keepalived documentation. - enum: - - NAT - - DR - - TUN - type: string - persistenceTimeout: - description: |- - PersistenceTimeout specify a timeout value for persistent connections in - seconds. If not specified, defaults to 360 (6 minutes). - type: integer - type: object - type: array - vrrpInstances: + Keepalived contains configuration options related to the "Keepalived" type + of load balancing. + properties: + virtualServers: + description: |- + Configuration options related to the virtual servers. This is an array + which allows to configure multiple load balancers. + items: + description: VirtualServer defines the configuration + options for a virtual server. + properties: + delayLoop: + description: |- + DelayLoop is the delay timer for check polling. If not specified, defaults to 0. + kubebuilder:validation:Minimum=0 + type: integer + ipAddress: + description: IPAddress is the virtual IP address + used by the virtual server. + minLength: 1 + type: string + lbAlgo: + default: rr + description: |- + LBAlgo is the load balancing algorithm. If not specified, defaults to rr. + Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further + details refer to keepalived documentation. + enum: + - rr + - wrr + - lc + - wlc + - lblc + - dh + - sh + - sed + - nq + type: string + lbKind: + default: DR + description: |- + LBKind is the load balancing kind. If not specified, defaults to DR. + Valid values are NAT DR TUN. For further details refer to keepalived documentation. + enum: + - NAT + - DR + - TUN + type: string + persistenceTimeoutSeconds: + description: |- + PersistenceTimeoutSeconds specify a timeout value for persistent connections in + seconds. If not specified, defaults to 360 (6 minutes). + kubebuilder:validation:Minimum=0 + type: integer + required: + - ipAddress + type: object + type: array + x-kubernetes-list-map-keys: + - ipAddress + x-kubernetes-list-type: map + vrrpInstances: + description: |- + Configuration options related to the VRRP. This is an array which allows + to configure multiple virtual IPs. + items: + description: VRRPInstance defines the configuration + options for a VRRP instance. + properties: + advertInterval: + default: 1 + description: |- + AdvertInterval is the advertisement interval in seconds. If not specified, + use 1 second + format: int32 + type: integer + authPass: + description: |- + AuthPass is the password for accessing vrrpd. This is not a security + feature but a way to prevent accidental misconfigurations. + Authpass must be 8 characters or less. + type: string + interface: + description: |- + Interface specifies the NIC used by the virtual router. If not specified, + k0s will use the interface that owns the default route. + type: string + name: + default: k0s-vip + description: |- + Name is the name of the VRRP instance. If not specified, defaults to + k0s-vip-. + type: string + virtualIPs: + description: |- + VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs + must be a CIDR as defined in RFC 4632 and RFC 4291. + items: + type: string + type: array + virtualRouterID: + default: 51 + description: |- + VirtualRouterID is the VRRP router ID. If not specified, defaults to 51. + VirtualRouterID must be in the range of 1-255, all the control plane + nodes must have the same VirtualRouterID. + Two clusters in the same network must not use the same VirtualRouterID. + format: int32 + maximum: 255 + minimum: 1 + type: integer + type: object + type: array + type: object + type: + default: Keepalived description: |- - Configuration options related to the VRRP. This is an array which allows - to configure multiple virtual IPs. - items: - description: VRRPInstance defines the configuration options - for a VRRP instance. - properties: - advertInterval: - default: 1 - description: |- - AdvertInterval is the advertisement interval in seconds. If not specified, - use 1 second - format: int32 - type: integer - authPass: - description: |- - AuthPass is the password for accessing vrrpd. This is not a security - feature but a way to prevent accidental misconfigurations. - Authpass must be 8 characters or less. - type: string - interface: - description: |- - Interface specifies the NIC used by the virtual router. If not specified, - k0s will use the interface that owns the default route. - type: string - name: - default: k0s-vip - description: |- - Name is the name of the VRRP instance. If not specified, defaults to - k0s-vip-. - type: string - virtualIPs: - description: |- - VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs - must be a CIDR as defined in RFC 4632 and RFC 4291. - items: - type: string - type: array - virtualRouterID: - default: 51 - description: |- - VirtualRouterID is the VRRP router ID. If not specified, defaults to 51. - VirtualRouterID must be in the range of 1-255, all the control plane - nodes must have the same VirtualRouterID. - Two clusters in the same network must not use the same VirtualRouterID. - format: int32 - maximum: 255 - minimum: 1 - type: integer - type: object - type: array + type indicates the type of the node-local load balancer to deploy on + worker nodes. Currently, the only supported type is "Keepalived". + enum: + - Keepalived + type: string type: object dualStack: description: DualStack defines network configuration for ipv4\ipv6 From 30966db75148570c975c8d5836eb7076b819a588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 6 May 2024 16:46:21 +0200 Subject: [PATCH 04/12] Implement VirtualServers inttest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- inttest/bootloose-alpine/Dockerfile | 3 +- inttest/cplb/cplb_test.go | 48 ++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/inttest/bootloose-alpine/Dockerfile b/inttest/bootloose-alpine/Dockerfile index b59b6a9486af..1fc57766755c 100644 --- a/inttest/bootloose-alpine/Dockerfile +++ b/inttest/bootloose-alpine/Dockerfile @@ -21,7 +21,8 @@ RUN apk add --no-cache \ curl \ haproxy \ nginx \ - inotify-tools + inotify-tools \ + ipvsadm # enable syslog and sshd RUN rc-update add syslog boot RUN rc-update add sshd default diff --git a/inttest/cplb/cplb_test.go b/inttest/cplb/cplb_test.go index 6a848b597ec9..1d75ff08c9eb 100644 --- a/inttest/cplb/cplb_test.go +++ b/inttest/cplb/cplb_test.go @@ -34,26 +34,33 @@ type keepalivedSuite struct { const haControllerConfig = ` spec: api: - externalAddress: %s + sans: + - %s network: controlPlaneLoadBalancing: enabled: true type: Keepalived + keepalived: vrrpInstances: - - virtualIPs: ["%s/24"] + - virtualIPs: ["%s/16"] authPass: "123456" + virtualServers: + - ipAddress: %s + nodeLocalLoadBalancing: + enabled: true + type: EnvoyProxy ` // SetupTest prepares the controller and filesystem, getting it into a consistent // state which we can run tests against. func (s *keepalivedSuite) TestK0sGetsUp() { - ipAddress := s.getLBAddress() + lb := s.getLBAddress() ctx := s.Context() var joinToken string for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second)) - s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, ipAddress, ipAddress)) + s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb, lb)) // Note that the token is intentionally empty for the first controller s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken)) @@ -86,18 +93,22 @@ func (s *keepalivedSuite) TestK0sGetsUp() { // Verify that all servers have the dummy interface for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { - s.checkDummy(ctx, s.ControllerNode(idx), ipAddress) + s.checkDummy(ctx, s.ControllerNode(idx), lb) } // Verify that only one controller has the VIP in eth0 count := 0 for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { - if s.hasVIP(ctx, s.ControllerNode(idx), ipAddress) { + if s.hasVIP(ctx, s.ControllerNode(idx), lb) { count++ } } - s.Require().Equal(1, count, "Expected only one controller to have the VIP") + s.Require().Equal(1, count, "Expected exactly one controller to have the VIP") + // Verify that the real servers are present in the ipvsadm output + for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { + s.validateRealServers(ctx, s.ControllerNode(idx), lb) + } } // getLBAddress returns the IP address of the controller 0 and it adds 100 to @@ -120,6 +131,27 @@ func (s *keepalivedSuite) getLBAddress() string { return fmt.Sprintf("%s.%d", strings.Join(parts[:3], "."), lastOctet) } +// validateRealServers checks that the real servers are present in the +// ipvsadm output. +func (s *keepalivedSuite) validateRealServers(ctx context.Context, node string, vip string) { + ssh, err := s.SSH(ctx, node) + s.Require().NoError(err) + defer ssh.Disconnect() + + servers := []string{} + for i := 0; i < s.BootlooseSuite.ControllerCount; i++ { + servers = append(servers, s.GetIPAddress(s.ControllerNode(i))) + } + + output, err := ssh.ExecWithOutput(ctx, "ipvsadm --save -n") + s.Require().NoError(err) + + for _, server := range servers { + s.Require().Contains(output, fmt.Sprintf("-a -t %s:6443 -r %s", vip, server), "Controller %s is missing a server in ipvsadm", node) + } + +} + // checkDummy checks that the dummy interface is present on the given node and // that it has only the virtual IP address. func (s *keepalivedSuite) checkDummy(ctx context.Context, node string, vip string) { @@ -146,7 +178,7 @@ func (s *keepalivedSuite) hasVIP(ctx context.Context, node string, vip string) b output, err := ssh.ExecWithOutput(ctx, "ip --oneline addr show eth0") s.Require().NoError(err) - return strings.Contains(output, fmt.Sprintf("inet %s/24", vip)) + return strings.Contains(output, fmt.Sprintf("inet %s/16", vip)) } // TestKeepAlivedSuite runs the keepalived test suite. It verifies that the From d61005e7523cfd7dc060246573fbe473c5a62f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 6 May 2024 16:55:20 +0200 Subject: [PATCH 05/12] Add VirtualServer docs and fix Keepalived subtype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- docs/configuration.md | 32 ++++++++++++++++--- docs/cplb.md | 73 ++++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c694750d79dd..8b821238c2ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -304,14 +304,24 @@ node-local load balancing. Configuration options related to k0s's [control plane load balancing] feature -| Element | Description | -| --------------- | ----------------------------------------------------------------------------------------------------------- | -| `enabled` | Indicates if control plane load balancing should be enabled. Default: `false`. | -| `vrrpInstances` | Configuration options related to the VRRP. This is an array which allows to configure multiple virtual IPs. | +| Element | Description | +| ------------ | ------------------------------------------------------------------------------------------- | +| `enabled` | Indicates if control plane load balancing should be enabled. Default: `false`. | +| `type` | Indicates the backend for CPLB. If this isn't defined to `Keepalived`, CPLB will not start. | +| `keepalived` | Contains the keepalived configuration. | [control plane load balancing]: cplb.md -##### `spec.network.controlPlaneLoadBalancing.VRRPInstances` +##### `spec.network.controlPlaneLoadBalancing.Keepalived` + +Configuration options related to keepalived in [control plane load balancing] + +| Element | Description | +| ---------------- | ----------------------------------------------------------------------------------------------------------- | +| `vrrpInstances` | Configuration options related to the VRRP. This is an array which allows to configure multiple virtual IPs. | +| `virtualServers` | Configuration options related LoadBalancing. This is an array which allows to configure multiple LBs. | + +##### `spec.network.controlPlaneLoadBalancing.keepalived.vrrpInstances` Configuration options required for using VRRP to configure VIPs in control plane load balancing. @@ -324,6 +334,18 @@ Configuration options required for using VRRP to configure VIPs in control plane | `advertInterval` | Advertisement interval in seconds. Default: `1`. | | `authPass` | The password used for accessing vrrpd. This field is mandatory and must be under 8 characters long | +##### `spec.network.controlPlaneLoadBalancing.keepalived.virtualServers` + +Configuration options required for using VRRP to configure VIPs in control plane load balancing. + +| Element | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `ipAddress` | The load balancer's listen address. | +| `delayLoop` | Delay timer for health check polling in seconds. Default: `0`. | +| `lbAlgo` | Algorithm used by keepalived. Supported algorithms: `rr`, `wrr`, `lc`, `wlc`, `lblc`, `dh`, `sh`, `sed`, `nq`. Default: `rr`. | +| `lbKind` | Kind of ipvs load balancer. Supported values: `NAT`, `DR`, `TUN` Default: `DR`. | +| `persistenceTimeout` | Timeout value for persistent connections in seconds. Default: `360` (6 minutes). | + ### `spec.controllerManager` | Element | Description | diff --git a/docs/cplb.md b/docs/cplb.md index fc8614eea230..1911f12d25a4 100644 --- a/docs/cplb.md +++ b/docs/cplb.md @@ -3,17 +3,23 @@ For clusters that don't have an [externally managed load balancer](high-availability.md#load-balancer) for the k0s control plane, there is another option to get a highly available control plane called control plane load balancing (CPLB). -CPLB allows automatic assigned of predefined IP addresses using VRRP across masters. +CPLB has two features that are independent, but normally will be used together: VRRP Instances, which allows +automatic assignation of predefined IP addresses using VRRP across control plane nodes. VirtualServers allows to +do Load Balancing to the other control plane nodes. + +This feature is intended to be used for external traffic. This feature is fully compatible with +[node-local load balancing (NLLB)](nllb.md) which means CPLB can be used for external traffic and NLLB for +internal traffic at the same time. ## Technical functionality -The k0s control plane load balancer provides k0s with virtual IPs on each -controller node. This allows the control plane to be highly available using -VRRP (Virtual Router Redundancy Protocol) as long as the network -infrastructure allows multicast and GARP. +The k0s control plane load balancer provides k0s with virtual IPs and TCP +load Balancing on each controller node. This allows the control plane to +be highly available using VRRP (Virtual Router Redundancy Protocol) and +IPVS long as the network infrastructure allows multicast and GARP. [Keepalived](https://www.keepalived.org/) is the only load balancer that is -supported so far and currently there are no plans to support other alternatives. +supported so far. Currently there are no plans to support other alternatives. ## VRRP Instances @@ -46,19 +52,24 @@ following: These do not provide any sort of security against ill-intentioned attacks, they are safety features to prevent accidental conflicts between VRRP instances in the same network segment. +* If `VirtualServers` are used, the cluster configuration mustn't specify a non-empty + [`spec.api.externalAddress`][specapi]. If only `VRRPInstances` are specified, a + non-empty [`spec.api.externalAddress`][specapi] may be specified. Add the following to the cluster configuration (`k0s.yaml`): ```yaml spec: - api: - externalAddress: # This isn't a requirement, but it's a common use case. network: controlPlaneLoadBalancing: enabled: true - vrrpInstances: - - virtualIPs: ["/ + type: Keepalived + keepalived: + vrrpInstances: + - virtualIPs: ["/ + virtualServers: + - ipAddress: "ipAddress" ``` Or alternatively, if using [`k0sctl`](k0sctl-install.md), add the following to @@ -69,24 +80,30 @@ spec: k0s: config: spec: - api: - externalAddress: # This isn't a requirement, but it's a common use case. network: controlPlaneLoadBalancing: enabled: true - vrrpInstances: - - virtualIPs: ["/"] - authPass: + type: Keepalived + keepalived: + vrrpInstances: + - virtualIPs: ["/"] + authPass: + virtualServers: + - ipAddress: "" ``` Because this is a feature intended to configure the apiserver, CPLB noes not support dynamic configuration and in order to make changes you need to restart the k0s controllers to make changes. +[specapi]: configuration.md#specapi + ## Full example using `k0sctl` The following example shows a full `k0sctl` configuration file featuring three -controllers and three workers with control plane load balancing enabled: +controllers and three workers with control plane load balancing enabled. +Additionally it defines [spec.api.sans](configuration.md#specapi) so that the +kube-apiserver certificate is valid for the virtual IP: ```yaml apiVersion: k0sctl.k0sproject.io/v1beta1 @@ -142,13 +159,18 @@ spec: config: spec: api: - externalAddress: 192.168.122.200 + sans: + - 192.168.122.200 network: controlPlaneLoadBalancing: enabled: true - vrrpInstances: - - virtualIPs: ["192.168.122.200/24"] - authPass: Example + type: Keepalived: + keepalived: + vrrpInstances: + - virtualIPs: ["192.168.122.200/24"] + authPass: Example + virtualServers: + - ipAddress: "" ``` Save the above configuration into a file called `k0sctl.yaml` and apply it in @@ -319,6 +341,15 @@ controller-1 controller-2 2: eth0 inet 192.168.122.87/24 brd 192.168.122.255 scope global dynamic noprefixroute eth0\ valid_lft 2182sec preferred_lft 2182sec 3: dummyvip0 inet 192.168.122.200/32 scope global dummyvip0\ valid_lft forever preferred_lft forever + +$ for i in controller-{0..2} ; do echo $i ; ipvsadm --save -n; done +IP Virtual Server version 1.2.1 (size=4096) +Prot LocalAddress:Port Scheduler Flags + -> RemoteAddress:Port Forward Weight ActiveConn InActConn +TCP 192.168.122.200:6443 rr persistent 360 + -> 192.168.122.185:6443 Route 1 0 0 + -> 192.168.122.87:6443 Route 1 0 0 + -> 192.168.122.122:6443 Route 1 0 0 ```` And the cluster will be working normally: From 261936465c7feeb4ab1383550224fa98b854291c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Mon, 6 May 2024 17:19:01 +0200 Subject: [PATCH 06/12] Automatically add virtualIPs to apiserver SANs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- cmd/controller/certificates.go | 14 ++++++++++++++ docs/cplb.md | 2 -- inttest/cplb/cplb_test.go | 5 +---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/controller/certificates.go b/cmd/controller/certificates.go index 22680503df0a..a46ae388dfd8 100644 --- a/cmd/controller/certificates.go +++ b/cmd/controller/certificates.go @@ -182,6 +182,20 @@ func (c *Certificates) Init(ctx context.Context) error { hostnames = append(hostnames, localIPs...) hostnames = append(hostnames, c.ClusterSpec.API.Sans()...) + // Add to SANs the IPs from the control plane load balancer + cplb := c.ClusterSpec.Network.ControlPlaneLoadBalancing + if cplb != nil && cplb.Enabled && cplb.Keepalived != nil { + for _, v := range cplb.Keepalived.VRRPInstances { + for _, vip := range v.VirtualIPs { + ip, _, err := net.ParseCIDR(vip) + if err != nil { + return fmt.Errorf("error parsing virtualIP %s: %w", vip, err) + } + hostnames = append(hostnames, ip.String()) + } + } + } + internalAPIAddress, err := c.ClusterSpec.Network.InternalAPIAddresses() if err != nil { return err diff --git a/docs/cplb.md b/docs/cplb.md index 1911f12d25a4..4e907a2cfffe 100644 --- a/docs/cplb.md +++ b/docs/cplb.md @@ -102,8 +102,6 @@ the k0s controllers to make changes. The following example shows a full `k0sctl` configuration file featuring three controllers and three workers with control plane load balancing enabled. -Additionally it defines [spec.api.sans](configuration.md#specapi) so that the -kube-apiserver certificate is valid for the virtual IP: ```yaml apiVersion: k0sctl.k0sproject.io/v1beta1 diff --git a/inttest/cplb/cplb_test.go b/inttest/cplb/cplb_test.go index 1d75ff08c9eb..477d6c66112a 100644 --- a/inttest/cplb/cplb_test.go +++ b/inttest/cplb/cplb_test.go @@ -33,9 +33,6 @@ type keepalivedSuite struct { const haControllerConfig = ` spec: - api: - sans: - - %s network: controlPlaneLoadBalancing: enabled: true @@ -60,7 +57,7 @@ func (s *keepalivedSuite) TestK0sGetsUp() { for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ { s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second)) - s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb, lb)) + s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb)) // Note that the token is intentionally empty for the first controller s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken)) From 9175b9fdebbb0db36da5eb9a1f0ea734f884f846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 10:15:49 +0200 Subject: [PATCH 07/12] Remove VRRPInstances.Name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The field wasn't required and didn't serve any actual purpose, so remove it and auto generate it always. Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- docs/configuration.md | 1 - pkg/apis/k0s/v1beta1/cplb.go | 9 --------- pkg/apis/k0s/v1beta1/cplb_test.go | 6 ------ pkg/component/controller/cplb_unix.go | 12 ++++++------ .../k0s.k0sproject.io_clusterconfigs.yaml | 6 ------ 5 files changed, 6 insertions(+), 28 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8b821238c2ff..84a40094e4f5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -327,7 +327,6 @@ Configuration options required for using VRRP to configure VIPs in control plane | Element | Description | | ----------------- | ----------------------------------------------------------------------------------------------------------------- | -| `name` | The name of the VRRP instance. If omitted it generates a predictive name shared across all nodes. | | `virtualIPs` | A list of the CIDRs handled by the VRRP instance. | | `interface` | The interface used by each VRRPInstance. If undefined k0s will try to auto detect it based on the default gateway | | `virtualRouterId` | Virtual router ID for the instance. Default: `51` | diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index ee55a3ce316e..f7cd5847ec61 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -69,11 +69,6 @@ type VRRPInstances []VRRPInstance // VRRPInstance defines the configuration options for a VRRP instance. type VRRPInstance struct { - // Name is the name of the VRRP instance. If not specified, defaults to - // k0s-vip-. - //+kubebuilder:default=k0s-vip - Name string `json:"name,omitempty"` - // VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs // must be a CIDR as defined in RFC 4632 and RFC 4291. VirtualIPs VirtualIPs `json:"virtualIPs,omitempty"` @@ -112,10 +107,6 @@ func (k *KeepalivedSpec) validateVRRPInstances(getDefaultNICFn func() (string, e getDefaultNICFn = getDefaultNIC } for i := range k.VRRPInstances { - if k.VRRPInstances[i].Name == "" { - k.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) - } - if k.VRRPInstances[i].Interface == "" { nic, err := getDefaultNICFn() if err != nil { diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index c42c6d28161a..a3202115e59d 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -48,7 +48,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { }, expectedVRRPs: []VRRPInstance{ { - Name: "k0s-vip-0", VirtualRouterID: ptr.To(int32(defaultVirtualRouterID)), Interface: "fake-nic-0", VirtualIPs: []string{"192.168.1.1/24"}, @@ -56,7 +55,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { AuthPass: "123456", }, { - Name: "k0s-vip-1", VirtualRouterID: ptr.To(int32(defaultVirtualRouterID + 1)), Interface: "fake-nic-0", VirtualIPs: []string{"192.168.1.1/24"}, @@ -70,7 +68,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { name: "valid instance no overrides", vrrps: []VRRPInstance{ { - Name: "test", VirtualRouterID: ptr.To(int32(1)), Interface: "eth0", VirtualIPs: []string{"192.168.1.1/24"}, @@ -80,7 +77,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { }, expectedVRRPs: []VRRPInstance{ { - Name: "test", VirtualRouterID: ptr.To(int32(1)), Interface: "eth0", VirtualIPs: []string{"192.168.1.1/24"}, @@ -93,7 +89,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { name: "No password", vrrps: []VRRPInstance{ { - Name: "test", VirtualRouterID: ptr.To(int32(1)), Interface: "eth0", VirtualIPs: []string{"192.168.1.1/24"}, @@ -135,7 +130,6 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { s.T().Log(k.VRRPInstances) s.Require().Equal(len(tt.expectedVRRPs), len(k.VRRPInstances), "Expected and actual VRRPInstances length mismatch") for i := 0; i < len(tt.expectedVRRPs); i++ { - s.Require().Equal(tt.expectedVRRPs[i].Name, k.VRRPInstances[i].Name, "Name mismatch") s.Require().Equal(tt.expectedVRRPs[i].Interface, k.VRRPInstances[i].Interface, "Interface mismatch") s.Require().Equal(*tt.expectedVRRPs[i].VirtualRouterID, *k.VRRPInstances[i].VirtualRouterID, "Virtual router ID mismatch") s.Require().Equal(*tt.expectedVRRPs[i].AdvertInterval, *k.VRRPInstances[i].AdvertInterval, "Virtual router ID mismatch") diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index 2075499af1d0..305cc2e9673e 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -358,26 +358,26 @@ type keepalivedConfig struct { const dummyLinkName = "dummyvip0" var keepalivedConfigTemplate = template.Must(template.New("keepalived").Parse(` -{{ range .VRRPInstances }} -vrrp_instance {{ .Name }} { +{{ range $i, $instance := .VRRPInstances }} +vrrp_instance k0s-vip-{{$i}} { # All servers must have state BACKUP so that when a new server comes up # it doesn't perform a failover. This must be combined with the priority. state BACKUP # Make sure the interface is aligned with your server's network interface interface {{ .Interface }} # The virtual router ID must be unique to each VRRP instance that you define - virtual_router_id {{ .VirtualRouterID }} + virtual_router_id {{ $instance.VirtualRouterID }} # All servers have the same priority so that when a new one comes up we don't # do a failover priority 200 # advertisement interval, 1 second by default - advert_int {{ .AdvertInterval }} + advert_int {{ $instance.AdvertInterval }} authentication { auth_type PASS - auth_pass {{ .AuthPass }} + auth_pass {{ $instance.AuthPass }} } virtual_ipaddress { - {{ range .VirtualIPs }} + {{ range $instance.VirtualIPs }} {{ . }} {{ end }} } diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index 85d0627bcc3d..a72a788cec1b 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -465,12 +465,6 @@ spec: Interface specifies the NIC used by the virtual router. If not specified, k0s will use the interface that owns the default route. type: string - name: - default: k0s-vip - description: |- - Name is the name of the VRRP instance. If not specified, defaults to - k0s-vip-. - type: string virtualIPs: description: |- VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs From 8d57249f31a1770cbe5548ff99f36773e602af40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 11:42:13 +0200 Subject: [PATCH 08/12] Remove ControlPlaneLoadBalancing from dynamic conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- pkg/apis/k0s/v1beta1/clusterconfig_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/apis/k0s/v1beta1/clusterconfig_types.go b/pkg/apis/k0s/v1beta1/clusterconfig_types.go index cdbebe68487c..493d943ab656 100644 --- a/pkg/apis/k0s/v1beta1/clusterconfig_types.go +++ b/pkg/apis/k0s/v1beta1/clusterconfig_types.go @@ -396,6 +396,7 @@ func (c *ClusterConfig) Validate() (errs []error) { // - StorageSpec // - Network.ServiceCIDR // - Network.ClusterDomain +// - Network.ControlPlaneLoadBalancing // - Install func (c *ClusterConfig) GetClusterWideConfig() *ClusterConfig { c = c.DeepCopy() @@ -405,6 +406,7 @@ func (c *ClusterConfig) GetClusterWideConfig() *ClusterConfig { if c.Spec.Network != nil { c.Spec.Network.ServiceCIDR = "" c.Spec.Network.ClusterDomain = "" + c.Spec.Network.ControlPlaneLoadBalancing = nil } c.Spec.Install = nil } From d5d7f1ab989166986c283f37b114c6815d5b61e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 13:18:34 +0200 Subject: [PATCH 09/12] Make CPLB.VirtualServers.DelayLoop metav1.Duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- pkg/apis/k0s/v1beta1/cplb.go | 12 +++- pkg/apis/k0s/v1beta1/cplb_test.go | 31 +++++++-- pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 1 + pkg/component/controller/cplb_unix.go | 13 +++- pkg/component/controller/cplb_unix_test.go | 69 +++++++++++++++++++ .../k0s.k0sproject.io_clusterconfigs.yaml | 4 +- 6 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 pkg/component/controller/cplb_unix_test.go diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index f7cd5847ec61..35e01fca4e44 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -20,6 +20,9 @@ import ( "errors" "fmt" "net" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Defaults are keepalived's defaults. @@ -158,8 +161,10 @@ type VirtualServer struct { // +kubebuilder:validation:MinLength=1 IPAddress string `json:"ipAddress"` // DelayLoop is the delay timer for check polling. If not specified, defaults to 0. + // DelayLoop is defined in microseconds. Further precision will be truncated without + // warnings. // kubebuilder:validation:Minimum=0 - DelayLoop int `json:"delayLoop,omitempty"` + DelayLoop metav1.Duration `json:"delayLoop,omitempty"` // LBAlgo is the load balancing algorithm. If not specified, defaults to rr. // Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further // details refer to keepalived documentation. @@ -250,8 +255,9 @@ func (k *KeepalivedSpec) validateVirtualServers() []error { errs = append(errs, errors.New("PersistenceTimeout must be a positive integer")) } - if k.VirtualServers[i].DelayLoop < 0 { - errs = append(errs, errors.New("DelayLoop must be a positive integer")) + k.VirtualServers[i].DelayLoop = metav1.Duration{Duration: k.VirtualServers[i].DelayLoop.Truncate(time.Microsecond)} + if k.VirtualServers[i].DelayLoop.Duration < time.Duration(0) { + errs = append(errs, errors.New("DelayLoop must be a positive")) } } return errs diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index a3202115e59d..eb9029cccf56 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -18,8 +18,10 @@ package v1beta1 import ( "testing" + "time" "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" ) @@ -163,14 +165,14 @@ func (s *CPLBSuite) TestValidateVirtualServers() { expectedVSS: []VirtualServer{ { IPAddress: "1.2.3.4", - DelayLoop: 0, + DelayLoop: metav1.Duration{Duration: 0}, LBAlgo: RRAlgo, LBKind: DRLBKind, PersistenceTimeoutSeconds: 360, }, { IPAddress: "1.2.3.5", - DelayLoop: 0, + DelayLoop: metav1.Duration{Duration: 0}, LBAlgo: RRAlgo, LBKind: DRLBKind, PersistenceTimeoutSeconds: 360, @@ -183,7 +185,7 @@ func (s *CPLBSuite) TestValidateVirtualServers() { vss: []VirtualServer{ { IPAddress: "1.2.3.4", - DelayLoop: 1, + DelayLoop: metav1.Duration{Duration: 1 * time.Second}, LBAlgo: WRRAlgo, LBKind: NATLBKind, PersistenceTimeoutSeconds: 100, @@ -192,7 +194,7 @@ func (s *CPLBSuite) TestValidateVirtualServers() { expectedVSS: []VirtualServer{ { IPAddress: "1.2.3.4", - DelayLoop: 1, + DelayLoop: metav1.Duration{Duration: 1 * time.Second}, LBAlgo: WRRAlgo, LBKind: NATLBKind, PersistenceTimeoutSeconds: 100, @@ -200,6 +202,25 @@ func (s *CPLBSuite) TestValidateVirtualServers() { }, wantErr: false, }, + { + name: "truncate DelayLoop", + vss: []VirtualServer{ + { + IPAddress: "1.2.3.4", + DelayLoop: metav1.Duration{Duration: 1234567 * time.Nanosecond}, + }, + }, + expectedVSS: []VirtualServer{ + { + IPAddress: "1.2.3.4", + DelayLoop: metav1.Duration{Duration: 1234 * time.Microsecond}, + LBAlgo: RRAlgo, + LBKind: DRLBKind, + PersistenceTimeoutSeconds: 360, + }, + }, + wantErr: false, + }, { name: "empty ip address", vss: []VirtualServer{{}}, @@ -236,7 +257,7 @@ func (s *CPLBSuite) TestValidateVirtualServers() { { name: "invalid delay loop", vss: []VirtualServer{{ - DelayLoop: -1, + DelayLoop: metav1.Duration{Duration: -1}, }}, wantErr: true, }, diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index cb2a69b71d16..3c74908deab4 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -1089,6 +1089,7 @@ func (in VirtualIPs) DeepCopy() VirtualIPs { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualServer) DeepCopyInto(out *VirtualServer) { *out = *in + out.DelayLoop = in.DelayLoop } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualServer. diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index 305cc2e9673e..b74efadc5b13 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -41,6 +41,7 @@ import ( "github.com/k0sproject/k0s/pkg/supervisor" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Keepalived is the controller for the keepalived process in the control plane load balancing @@ -357,7 +358,11 @@ type keepalivedConfig struct { const dummyLinkName = "dummyvip0" -var keepalivedConfigTemplate = template.Must(template.New("keepalived").Parse(` +var keepalivedConfigTemplate = template.Must(template.New("keepalived"). + Funcs(template.FuncMap{ + "delay_loop_str": delayLoopString, + }). + Parse(` {{ range $i, $instance := .VRRPInstances }} vrrp_instance k0s-vip-{{$i}} { # All servers must have state BACKUP so that when a new server comes up @@ -389,7 +394,7 @@ vrrp_instance k0s-vip-{{$i}} { {{ if gt (len $RealServers) 0 }} {{ range .VirtualServers }} virtual_server {{ .IPAddress }} {{ $APIServerPort }} { - delay_loop {{ .DelayLoop }} + delay_loop {{ delay_loop_str .DelayLoop }} lb_algo {{ .LBAlgo }} lb_kind {{ .LBKind }} persistence_timeout {{ .PersistenceTimeoutSeconds }} @@ -410,3 +415,7 @@ virtual_server {{ .IPAddress }} {{ $APIServerPort }} { {{ end }} {{ end }} `)) + +func delayLoopString(delayLoop metav1.Duration) string { + return fmt.Sprintf("%v", delayLoop.Duration.Seconds()) +} diff --git a/pkg/component/controller/cplb_unix_test.go b/pkg/component/controller/cplb_unix_test.go new file mode 100644 index 000000000000..c5ca3a505818 --- /dev/null +++ b/pkg/component/controller/cplb_unix_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + "time" + + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type CPLBUnixSuite struct { + suite.Suite +} + +func (s *CPLBUnixSuite) TestDelayLoopString() { + tests := []struct { + name string + duration metav1.Duration + output string + }{ + { + name: "2 seconds", + duration: metav1.Duration{Duration: 2 * time.Second}, + output: "2", + }, + { + name: "1234 microseconds", + duration: metav1.Duration{Duration: 1234 * time.Microsecond}, + output: "0.001234", + }, + { + name: "1.5 seconds", + duration: metav1.Duration{Duration: 1500 * time.Millisecond}, + output: "1.5", + }, + { + name: "2 hours", + duration: metav1.Duration{Duration: 2 * time.Hour}, + output: "7200", + }, + } + for _, tt := range tests { + s.T().Run(tt.name, func(t *testing.T) { + s.Equal(tt.output, delayLoopString(tt.duration)) + }) + } +} + +func TestCPLUnixSuite(t *testing.T) { + cplUnixSuite := &CPLBUnixSuite{} + + suite.Run(t, cplUnixSuite) +} diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index a72a788cec1b..2be5381791e3 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -392,8 +392,10 @@ spec: delayLoop: description: |- DelayLoop is the delay timer for check polling. If not specified, defaults to 0. + DelayLoop is defined in microseconds. Further precision will be truncated without + warnings. kubebuilder:validation:Minimum=0 - type: integer + type: string ipAddress: description: IPAddress is the virtual IP address used by the virtual server. From 0e5329973c866e4febecc686fbbac23d3d8b7414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 13:40:06 +0200 Subject: [PATCH 10/12] Addess multiple mistakes in code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- pkg/apis/k0s/v1beta1/cplb.go | 24 ++++--- pkg/component/controller/cplb_reconciler.go | 1 + pkg/component/controller/cplb_unix.go | 19 ++--- pkg/component/controller/cplb_unix_test.go | 69 ------------------- .../k0s.k0sproject.io_clusterconfigs.yaml | 13 ++-- 5 files changed, 30 insertions(+), 96 deletions(-) delete mode 100644 pkg/component/controller/cplb_unix_test.go diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index 35e01fca4e44..a40143ec1024 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -37,8 +37,8 @@ type ControlPlaneLoadBalancingSpec struct { // +optional Enabled bool `json:"enabled,omitempty"` - // type indicates the type of the node-local load balancer to deploy on - // worker nodes. Currently, the only supported type is "Keepalived". + // type indicates the type of the control plane load balancer to deploy on + // controller nodes. Currently, the only supported type is "Keepalived". // +kubebuilder:default=Keepalived // +optional Type CPLBType `json:"type,omitempty"` @@ -48,8 +48,8 @@ type ControlPlaneLoadBalancingSpec struct { Keepalived *KeepalivedSpec `json:"keepalived,omitempty"` } -// NllbType describes which type of load balancer should be deployed for the -// node-local load balancing. The default is [CPLBTypeKeepalived]. +// CPLBType describes which type of load balancer should be deployed for the +// control plane load balancing. The default is [CPLBTypeKeepalived]. // +kubebuilder:validation:Enum=Keepalived type CPLBType string @@ -176,9 +176,13 @@ type VirtualServer struct { // +kubebuilder:default=DR // +optional LBKind KeepalivedLBKind `json:"lbKind,omitempty"` - // PersistenceTimeoutSeconds specify a timeout value for persistent connections in - // seconds. If not specified, defaults to 360 (6 minutes). - // kubebuilder:validation:Minimum=0 + // PersistenceTimeoutSeconds specifies a timeout value for persistent + // connections in seconds. PersistentTimeoutSeconds must be in the range of + // 1-2678400 (31 days). If not specified, defaults to 360 (6 minutes). + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=2678400 + // +kubebuilder:default=360 + // +optional PersistenceTimeoutSeconds int `json:"persistenceTimeoutSeconds,omitempty"` } @@ -215,7 +219,7 @@ type RealServer struct { Weight int `json:"weight,omitempty"` } -// validateVRRPInstances validates existing configuration and sets the default +// validateVirtualServers validates existing configuration and sets the default // values of undefined fields. func (k *KeepalivedSpec) validateVirtualServers() []error { errs := []error{} @@ -251,8 +255,8 @@ func (k *KeepalivedSpec) validateVirtualServers() []error { if k.VirtualServers[i].PersistenceTimeoutSeconds == 0 { k.VirtualServers[i].PersistenceTimeoutSeconds = 360 - } else if k.VirtualServers[i].PersistenceTimeoutSeconds < 0 { - errs = append(errs, errors.New("PersistenceTimeout must be a positive integer")) + } else if k.VirtualServers[i].PersistenceTimeoutSeconds < 1 || k.VirtualServers[i].PersistenceTimeoutSeconds > 2678400 { + errs = append(errs, errors.New("PersistenceTimeout must be in the range of 1-2678400")) } k.VirtualServers[i].DelayLoop = metav1.Duration{Duration: k.VirtualServers[i].DelayLoop.Truncate(time.Microsecond)} diff --git a/pkg/component/controller/cplb_reconciler.go b/pkg/component/controller/cplb_reconciler.go index 5deaefb62b67..5a534c90b965 100644 --- a/pkg/component/controller/cplb_reconciler.go +++ b/pkg/component/controller/cplb_reconciler.go @@ -67,6 +67,7 @@ func (r *CPLBReconciler) Start() error { func (r *CPLBReconciler) Stop() { r.log.Info("Stopping CPLB reconciler") r.watchCancelFunc() + close(r.updateCh) } func (r *CPLBReconciler) watchAPIServers(ctx context.Context, clientSet kubernetes.Interface) { diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index b74efadc5b13..246cc8f37156 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -41,7 +41,6 @@ import ( "github.com/k0sproject/k0s/pkg/supervisor" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Keepalived is the controller for the keepalived process in the control plane load balancing @@ -322,7 +321,11 @@ func (k *Keepalived) watchReconcilerUpdates() { // Wait for the supervisor to start keepalived before // watching for endpoint changes process := k.supervisor.GetProcess() - for process == nil { + for i := 0; process == nil; i++ { + if i > 3 { + k.log.Error("failed to start keepalived, supervisor process is nil") + return + } k.log.Info("Waiting for keepalived to start") time.Sleep(5 * time.Second) process = k.supervisor.GetProcess() @@ -358,11 +361,7 @@ type keepalivedConfig struct { const dummyLinkName = "dummyvip0" -var keepalivedConfigTemplate = template.Must(template.New("keepalived"). - Funcs(template.FuncMap{ - "delay_loop_str": delayLoopString, - }). - Parse(` +var keepalivedConfigTemplate = template.Must(template.New("keepalived").Parse(` {{ range $i, $instance := .VRRPInstances }} vrrp_instance k0s-vip-{{$i}} { # All servers must have state BACKUP so that when a new server comes up @@ -394,7 +393,7 @@ vrrp_instance k0s-vip-{{$i}} { {{ if gt (len $RealServers) 0 }} {{ range .VirtualServers }} virtual_server {{ .IPAddress }} {{ $APIServerPort }} { - delay_loop {{ delay_loop_str .DelayLoop }} + delay_loop {{ .DelayLoop.Seconds }} lb_algo {{ .LBAlgo }} lb_kind {{ .LBKind }} persistence_timeout {{ .PersistenceTimeoutSeconds }} @@ -415,7 +414,3 @@ virtual_server {{ .IPAddress }} {{ $APIServerPort }} { {{ end }} {{ end }} `)) - -func delayLoopString(delayLoop metav1.Duration) string { - return fmt.Sprintf("%v", delayLoop.Duration.Seconds()) -} diff --git a/pkg/component/controller/cplb_unix_test.go b/pkg/component/controller/cplb_unix_test.go deleted file mode 100644 index c5ca3a505818..000000000000 --- a/pkg/component/controller/cplb_unix_test.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2024 k0s authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "testing" - "time" - - "github.com/stretchr/testify/suite" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type CPLBUnixSuite struct { - suite.Suite -} - -func (s *CPLBUnixSuite) TestDelayLoopString() { - tests := []struct { - name string - duration metav1.Duration - output string - }{ - { - name: "2 seconds", - duration: metav1.Duration{Duration: 2 * time.Second}, - output: "2", - }, - { - name: "1234 microseconds", - duration: metav1.Duration{Duration: 1234 * time.Microsecond}, - output: "0.001234", - }, - { - name: "1.5 seconds", - duration: metav1.Duration{Duration: 1500 * time.Millisecond}, - output: "1.5", - }, - { - name: "2 hours", - duration: metav1.Duration{Duration: 2 * time.Hour}, - output: "7200", - }, - } - for _, tt := range tests { - s.T().Run(tt.name, func(t *testing.T) { - s.Equal(tt.output, delayLoopString(tt.duration)) - }) - } -} - -func TestCPLUnixSuite(t *testing.T) { - cplUnixSuite := &CPLBUnixSuite{} - - suite.Run(t, cplUnixSuite) -} diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index 2be5381791e3..e575899d7c26 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -429,10 +429,13 @@ spec: - TUN type: string persistenceTimeoutSeconds: + default: 360 description: |- - PersistenceTimeoutSeconds specify a timeout value for persistent connections in - seconds. If not specified, defaults to 360 (6 minutes). - kubebuilder:validation:Minimum=0 + PersistenceTimeoutSeconds specifies a timeout value for persistent + connections in seconds. PersistentTimeoutSeconds must be in the range of + 1-2678400 (31 days). If not specified, defaults to 360 (6 minutes). + maximum: 2678400 + minimum: 1 type: integer required: - ipAddress @@ -491,8 +494,8 @@ spec: type: default: Keepalived description: |- - type indicates the type of the node-local load balancer to deploy on - worker nodes. Currently, the only supported type is "Keepalived". + type indicates the type of the control plane load balancer to deploy on + controller nodes. Currently, the only supported type is "Keepalived". enum: - Keepalived type: string From c22cbf32a2e8891b0cf84befa4c1c77485b2bee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 15:44:43 +0200 Subject: [PATCH 11/12] Remove unused RealServer struct type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- pkg/apis/k0s/v1beta1/cplb.go | 7 ------- pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 15 --------------- 2 files changed, 22 deletions(-) diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index a40143ec1024..0cea640a718b 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -212,13 +212,6 @@ const ( TUNLBKind KeepalivedLBKind = "TUN" ) -type RealServer struct { - // IPAddress is the IP address of the real server. - IPAddress string `json:"ipAddress"` - // Weight is the weight of the real server. If not specified, defaults to 1. - Weight int `json:"weight,omitempty"` -} - // validateVirtualServers validates existing configuration and sets the default // values of undefined fields. func (k *KeepalivedSpec) validateVirtualServers() []error { diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index 3c74908deab4..5360cd6ad1a8 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -890,21 +890,6 @@ func (in *NodeLocalLoadBalancing) DeepCopy() *NodeLocalLoadBalancing { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RealServer) DeepCopyInto(out *RealServer) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RealServer. -func (in *RealServer) DeepCopy() *RealServer { - if in == nil { - return nil - } - out := new(RealServer) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in RepositoriesSettings) DeepCopyInto(out *RepositoriesSettings) { { From 18d35835f15b9b8113a8473d18e2662e6bedc047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 7 May 2024 16:12:47 +0200 Subject: [PATCH 12/12] Fix documentation for Delayloop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tom Wieczorek Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- docs/configuration.md | 14 +++++++------- pkg/apis/k0s/v1beta1/cplb.go | 6 +++--- .../k0s.k0sproject.io_clusterconfigs.yaml | 5 ++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 84a40094e4f5..5c6dababa873 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -337,13 +337,13 @@ Configuration options required for using VRRP to configure VIPs in control plane Configuration options required for using VRRP to configure VIPs in control plane load balancing. -| Element | Description | -| ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `ipAddress` | The load balancer's listen address. | -| `delayLoop` | Delay timer for health check polling in seconds. Default: `0`. | -| `lbAlgo` | Algorithm used by keepalived. Supported algorithms: `rr`, `wrr`, `lc`, `wlc`, `lblc`, `dh`, `sh`, `sed`, `nq`. Default: `rr`. | -| `lbKind` | Kind of ipvs load balancer. Supported values: `NAT`, `DR`, `TUN` Default: `DR`. | -| `persistenceTimeout` | Timeout value for persistent connections in seconds. Default: `360` (6 minutes). | +| Element | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ipAddress` | The load balancer's listen address. | +| `delayLoop` | Delay timer for check polling. DelayLoop accepts microsecond precision. Further precision will be truncated without warnings. Example: `10s`. | +| `lbAlgo` | Algorithm used by keepalived. Supported algorithms: `rr`, `wrr`, `lc`, `wlc`, `lblc`, `dh`, `sh`, `sed`, `nq`. Default: `rr`. | +| `lbKind` | Kind of ipvs load balancer. Supported values: `NAT`, `DR`, `TUN` Default: `DR`. | +| `persistenceTimeoutSeconds` | Timeout value for persistent connections in seconds. Must be in the range of 1-2678400 (31 days). If not specified, defaults to 360 (6 minutes). | ### `spec.controllerManager` diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index 0cea640a718b..c11b1b77f55a 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -160,10 +160,10 @@ type VirtualServer struct { // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 IPAddress string `json:"ipAddress"` - // DelayLoop is the delay timer for check polling. If not specified, defaults to 0. - // DelayLoop is defined in microseconds. Further precision will be truncated without + // DelayLoop is the delay timer for check polling. DelayLoop accepts + // microsecond precision. Further precision will be truncated without // warnings. - // kubebuilder:validation:Minimum=0 + // +optional DelayLoop metav1.Duration `json:"delayLoop,omitempty"` // LBAlgo is the load balancing algorithm. If not specified, defaults to rr. // Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index e575899d7c26..2f9e4c6f115a 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -391,10 +391,9 @@ spec: properties: delayLoop: description: |- - DelayLoop is the delay timer for check polling. If not specified, defaults to 0. - DelayLoop is defined in microseconds. Further precision will be truncated without + DelayLoop is the delay timer for check polling. DelayLoop accepts + microsecond precision. Further precision will be truncated without warnings. - kubebuilder:validation:Minimum=0 type: string ipAddress: description: IPAddress is the virtual IP address