From b9c88031f60457880195752f003e1415d77a3538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Tue, 30 Apr 2024 14:10:00 +0200 Subject: [PATCH] Encapsulate Keepalived config in a new type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the future CPLB may want to support other backends. Encapsulate all keepalived's configuration in a new struct, just like NLLB. Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- cmd/controller/controller.go | 5 +- inttest/cplb/cplb_test.go | 12 +- pkg/apis/k0s/v1beta1/cplb.go | 86 ++++--- pkg/apis/k0s/v1beta1/cplb_test.go | 29 ++- pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go | 42 +++- pkg/component/controller/cplb_unix.go | 2 +- .../k0s.k0sproject.io_clusterconfigs.yaml | 210 ++++++++++-------- 7 files changed, 223 insertions(+), 163 deletions(-) diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index d0cdd653fb8f..01d425548a55 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -232,10 +232,13 @@ func (c *command) start(ctx context.Context) error { if c.SingleNode { return errors.New("control plane load balancing cannot be used in a single-node cluster") } + if cplb.Type != v1beta1.CPLBTypeKeepalived { + return errors.New("invalid control plane load balancing type. Only 'keepalived' is supported") + } nodeComponents.Add(ctx, &controller.Keepalived{ K0sVars: c.K0sVars, - Config: cplb, + Config: cplb.Keepalived, DetailedLogging: c.Debug, KubeConfigPath: c.K0sVars.AdminKubeConfigPath, APISpec: nodeConfig.Spec.API, diff --git a/inttest/cplb/cplb_test.go b/inttest/cplb/cplb_test.go index 0bb09633828a..1d75ff08c9eb 100644 --- a/inttest/cplb/cplb_test.go +++ b/inttest/cplb/cplb_test.go @@ -39,11 +39,13 @@ spec: network: controlPlaneLoadBalancing: enabled: true - vrrpInstances: - - virtualIPs: ["%s/16"] - authPass: "123456" - virtualServers: - - ipAddress: %s + type: Keepalived + keepalived: + vrrpInstances: + - virtualIPs: ["%s/16"] + authPass: "123456" + virtualServers: + - ipAddress: %s nodeLocalLoadBalancing: enabled: true type: EnvoyProxy diff --git a/pkg/apis/k0s/v1beta1/cplb.go b/pkg/apis/k0s/v1beta1/cplb.go index 033ac8a1c8ee..759c2c76cf8a 100644 --- a/pkg/apis/k0s/v1beta1/cplb.go +++ b/pkg/apis/k0s/v1beta1/cplb.go @@ -34,6 +34,28 @@ type ControlPlaneLoadBalancingSpec struct { // +optional Enabled bool `json:"enabled,omitempty"` + // type indicates the type of the node-local load balancer to deploy on + // worker nodes. Currently, the only supported type is "Keepalived". + // +kubebuilder:default=Keepalived + // +optional + Type CPLBType `json:"type,omitempty"` + + // Keepalived contains configuration options related to the "Keepalived" type + // of load balancing. + Keepalived *KeepalivedSpec `json:"keepalived,omitempty"` +} + +// NllbType describes which type of load balancer should be deployed for the +// node-local load balancing. The default is [CPLBTypeKeepalived]. +// +kubebuilder:validation:Enum=keepalived +type CPLBType string + +const ( + // CPLBTypeKeepalived selects Keepalived as the backing load balancer. + CPLBTypeKeepalived CPLBType = "Keepalived" +) + +type KeepalivedSpec struct { // Configuration options related to the VRRP. This is an array which allows // to configure multiple virtual IPs. VRRPInstances VRRPInstances `json:"vrrpInstances,omitempty"` @@ -84,46 +106,46 @@ type VirtualIPs []string // ValidateVRRPInstances validates existing configuration and sets the default // values of undefined fields. -func (c *ControlPlaneLoadBalancingSpec) ValidateVRRPInstances(getDefaultNICFn func() (string, error)) error { +func (k *KeepalivedSpec) ValidateVRRPInstances(getDefaultNICFn func() (string, error)) error { if getDefaultNICFn == nil { getDefaultNICFn = getDefaultNIC } - for i := range c.VRRPInstances { - if c.VRRPInstances[i].Name == "" { - c.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) + for i := range k.VRRPInstances { + if k.VRRPInstances[i].Name == "" { + k.VRRPInstances[i].Name = fmt.Sprintf("k0s-vip-%d", i) } - if c.VRRPInstances[i].Interface == "" { + if k.VRRPInstances[i].Interface == "" { nic, err := getDefaultNICFn() if err != nil { return fmt.Errorf("failed to get default NIC: %w", err) } - c.VRRPInstances[i].Interface = nic + k.VRRPInstances[i].Interface = nic } - if c.VRRPInstances[i].VirtualRouterID == nil { + if k.VRRPInstances[i].VirtualRouterID == nil { vrid := int32(defaultVirtualRouterID + i) - c.VRRPInstances[i].VirtualRouterID = &vrid - } else if *c.VRRPInstances[i].VirtualRouterID < 0 || *c.VRRPInstances[i].VirtualRouterID > 255 { + k.VRRPInstances[i].VirtualRouterID = &vrid + } else if *k.VRRPInstances[i].VirtualRouterID < 0 || *k.VRRPInstances[i].VirtualRouterID > 255 { return errors.New("VirtualRouterID must be in the range of 1-255") } - if c.VRRPInstances[i].AdvertInterval == nil { + if k.VRRPInstances[i].AdvertInterval == nil { advInt := int32(defaultAdvertInterval) - c.VRRPInstances[i].AdvertInterval = &advInt + k.VRRPInstances[i].AdvertInterval = &advInt } - if c.VRRPInstances[i].AuthPass == "" { + if k.VRRPInstances[i].AuthPass == "" { return errors.New("AuthPass must be defined") } - if len(c.VRRPInstances[i].AuthPass) > 8 { + if len(k.VRRPInstances[i].AuthPass) > 8 { return errors.New("AuthPass must be 8 characters or less") } - if len(c.VRRPInstances[i].VirtualIPs) == 0 { + if len(k.VRRPInstances[i].VirtualIPs) == 0 { return errors.New("VirtualIPs must be defined") } - for _, vip := range c.VRRPInstances[i].VirtualIPs { + for _, vip := range k.VRRPInstances[i].VirtualIPs { if _, _, err := net.ParseCIDR(vip); err != nil { return fmt.Errorf("VirtualIPs must be a CIDR. Got: %s", vip) } @@ -186,44 +208,44 @@ type RealServer struct { Weight int `json:"weight,omitempty"` } -func (c *ControlPlaneLoadBalancingSpec) ValidateVirtualServers() error { - for i := range c.VirtualServers { - if c.VirtualServers[i].IPAddress == "" { +func (k *KeepalivedSpec) ValidateVirtualServers() error { + for i := range k.VirtualServers { + if k.VirtualServers[i].IPAddress == "" { return errors.New("IPAddress must be defined") } - if net.ParseIP(c.VirtualServers[i].IPAddress) == nil { - return fmt.Errorf("invalid IP address: %s", c.VirtualServers[i].IPAddress) + if net.ParseIP(k.VirtualServers[i].IPAddress) == nil { + return fmt.Errorf("invalid IP address: %s", k.VirtualServers[i].IPAddress) } - if c.VirtualServers[i].LBAlgo == "" { - c.VirtualServers[i].LBAlgo = RRAlgo + if k.VirtualServers[i].LBAlgo == "" { + k.VirtualServers[i].LBAlgo = RRAlgo } else { - switch c.VirtualServers[i].LBAlgo { + switch k.VirtualServers[i].LBAlgo { case RRAlgo, WRRAlgo, LCAlgo, WLCAlgo, LBLCAlgo, DHAlgo, SHAlgo, SEDAlgo, NQAlgo: // valid LBAlgo default: - return fmt.Errorf("invalid LBAlgo: %s ", c.VirtualServers[i].LBAlgo) + return fmt.Errorf("invalid LBAlgo: %s ", k.VirtualServers[i].LBAlgo) } } - if c.VirtualServers[i].LBKind == "" { - c.VirtualServers[i].LBKind = DRLBKind + if k.VirtualServers[i].LBKind == "" { + k.VirtualServers[i].LBKind = DRLBKind } else { - switch c.VirtualServers[i].LBKind { + switch k.VirtualServers[i].LBKind { case NATLBKind, DRLBKind, TUNLBKind: // valid LBKind default: - return fmt.Errorf("invalid LBKind: %s ", c.VirtualServers[i].LBKind) + return fmt.Errorf("invalid LBKind: %s ", k.VirtualServers[i].LBKind) } } - if c.VirtualServers[i].PersistenceTimeout == 0 { - c.VirtualServers[i].PersistenceTimeout = 360 - } else if c.VirtualServers[i].PersistenceTimeout < 0 { + if k.VirtualServers[i].PersistenceTimeout == 0 { + k.VirtualServers[i].PersistenceTimeout = 360 + } else if k.VirtualServers[i].PersistenceTimeout < 0 { return errors.New("PersistenceTimeout must be a positive integer") } - if c.VirtualServers[i].DelayLoop < 0 { + if k.VirtualServers[i].DelayLoop < 0 { return errors.New("DelayLoop must be a positive integer") } } diff --git a/pkg/apis/k0s/v1beta1/cplb_test.go b/pkg/apis/k0s/v1beta1/cplb_test.go index 7a8a46cd7f76..0b0c4bd67ead 100644 --- a/pkg/apis/k0s/v1beta1/cplb_test.go +++ b/pkg/apis/k0s/v1beta1/cplb_test.go @@ -28,7 +28,6 @@ type CPLBSuite struct { } func (s *CPLBSuite) TestValidateVRRPInstances() { - tests := []struct { name string vrrps []VRRPInstance @@ -125,21 +124,21 @@ func (s *CPLBSuite) TestValidateVRRPInstances() { for _, tt := range tests { s.Run(tt.name, func() { - elb := &ControlPlaneLoadBalancingSpec{ + k := &KeepalivedSpec{ VRRPInstances: tt.vrrps, } - err := elb.ValidateVRRPInstances(returnNIC) + err := k.ValidateVRRPInstances(returnNIC) if tt.wantErr { s.Require().Errorf(err, "Test case %s expected error. Got none", tt.name) } else { s.Require().NoErrorf(err, "Test case %s expected no error. Got: %v", tt.name, err) - s.T().Log(elb.VRRPInstances) - s.Require().Equal(len(tt.expectedVRRPs), len(elb.VRRPInstances), "Expected and actual VRRPInstances length mismatch") + s.T().Log(k.VRRPInstances) + s.Require().Equal(len(tt.expectedVRRPs), len(k.VRRPInstances), "Expected and actual VRRPInstances length mismatch") for i := 0; i < len(tt.expectedVRRPs); i++ { - s.Require().Equal(tt.expectedVRRPs[i].Name, elb.VRRPInstances[i].Name, "Name mismatch") - s.Require().Equal(tt.expectedVRRPs[i].Interface, elb.VRRPInstances[i].Interface, "Interface mismatch") - s.Require().Equal(*tt.expectedVRRPs[i].VirtualRouterID, *elb.VRRPInstances[i].VirtualRouterID, "Virtual router ID mismatch") - s.Require().Equal(*tt.expectedVRRPs[i].AdvertInterval, *elb.VRRPInstances[i].AdvertInterval, "Virtual router ID mismatch") + s.Require().Equal(tt.expectedVRRPs[i].Name, k.VRRPInstances[i].Name, "Name mismatch") + s.Require().Equal(tt.expectedVRRPs[i].Interface, k.VRRPInstances[i].Interface, "Interface mismatch") + s.Require().Equal(*tt.expectedVRRPs[i].VirtualRouterID, *k.VRRPInstances[i].VirtualRouterID, "Virtual router ID mismatch") + s.Require().Equal(*tt.expectedVRRPs[i].AdvertInterval, *k.VRRPInstances[i].AdvertInterval, "Virtual router ID mismatch") } } }) @@ -250,17 +249,17 @@ func (s *CPLBSuite) TestValidateVirtualServers() { } for _, tt := range tests { s.Run(tt.name, func() { - cplb := &ControlPlaneLoadBalancingSpec{VirtualServers: tt.vss} - err := cplb.ValidateVirtualServers() + k := &KeepalivedSpec{VirtualServers: tt.vss} + err := k.ValidateVirtualServers() if tt.wantErr { s.Require().Errorf(err, "Test case %s expected error. Got none", tt.name) } else { s.Require().NoErrorf(err, "Tedst case %s expected no error. Got: %v", tt.name, err) for i := range tt.expectedVSS { - s.Require().Equal(tt.expectedVSS[i].DelayLoop, cplb.VirtualServers[i].DelayLoop, "DelayLoop mismatch") - s.Require().Equal(tt.expectedVSS[i].LBAlgo, cplb.VirtualServers[i].LBAlgo, "LBalgo mismatch") - s.Require().Equal(tt.expectedVSS[i].LBKind, cplb.VirtualServers[i].LBKind, "LBKind mismatch") - s.Require().Equal(tt.expectedVSS[i].PersistenceTimeout, cplb.VirtualServers[i].PersistenceTimeout, "PersistenceTimeout mismatch") + s.Require().Equal(tt.expectedVSS[i].DelayLoop, k.VirtualServers[i].DelayLoop, "DelayLoop mismatch") + s.Require().Equal(tt.expectedVSS[i].LBAlgo, k.VirtualServers[i].LBAlgo, "LBalgo mismatch") + s.Require().Equal(tt.expectedVSS[i].LBKind, k.VirtualServers[i].LBKind, "LBKind mismatch") + s.Require().Equal(tt.expectedVSS[i].PersistenceTimeout, k.VirtualServers[i].PersistenceTimeout, "PersistenceTimeout mismatch") } } }) diff --git a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go index f76903ac87e6..2a252ec1c3f4 100644 --- a/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/k0s/v1beta1/zz_generated.deepcopy.go @@ -387,17 +387,10 @@ func (in *ClusterTelemetry) DeepCopy() *ClusterTelemetry { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlPlaneLoadBalancingSpec) DeepCopyInto(out *ControlPlaneLoadBalancingSpec) { *out = *in - if in.VRRPInstances != nil { - in, out := &in.VRRPInstances, &out.VRRPInstances - *out = make(VRRPInstances, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.VirtualServers != nil { - in, out := &in.VirtualServers, &out.VirtualServers - *out = make(VirtualServers, len(*in)) - copy(*out, *in) + if in.Keepalived != nil { + in, out := &in.Keepalived, &out.Keepalived + *out = new(KeepalivedSpec) + (*in).DeepCopyInto(*out) } } @@ -657,6 +650,33 @@ func (in *InstallSpec) DeepCopy() *InstallSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeepalivedSpec) DeepCopyInto(out *KeepalivedSpec) { + *out = *in + if in.VRRPInstances != nil { + in, out := &in.VRRPInstances, &out.VRRPInstances + *out = make(VRRPInstances, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VirtualServers != nil { + in, out := &in.VirtualServers, &out.VirtualServers + *out = make(VirtualServers, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeepalivedSpec. +func (in *KeepalivedSpec) DeepCopy() *KeepalivedSpec { + if in == nil { + return nil + } + out := new(KeepalivedSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KineConfig) DeepCopyInto(out *KineConfig) { *out = *in diff --git a/pkg/component/controller/cplb_unix.go b/pkg/component/controller/cplb_unix.go index 9841a2d97ec3..af15f19d585b 100644 --- a/pkg/component/controller/cplb_unix.go +++ b/pkg/component/controller/cplb_unix.go @@ -46,7 +46,7 @@ import ( // Keepalived is the controller for the keepalived process in the control plane load balancing type Keepalived struct { K0sVars *config.CfgVars - Config *k0sAPI.ControlPlaneLoadBalancingSpec + Config *k0sAPI.KeepalivedSpec DetailedLogging bool uid int supervisor *supervisor.Supervisor diff --git a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml index 5bde3005ef68..248c5c261c6f 100644 --- a/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml +++ b/static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml @@ -372,106 +372,120 @@ spec: Indicates if control plane load balancing should be enabled. Default: false type: boolean - virtualServers: + keepalived: description: |- - Configuration options related to the virtual servers. This is an array - which allows to configure multiple load balancers. - items: - description: VirtualServer defines the configuration options - for a virtual server. - properties: - delayLoop: - description: DelayLoop is the delay timer for check - polling. If not specified, defaults to 0. - type: integer - ipAddress: - description: IPAddress is the virtual IP address used - by the virtual server. - type: string - lbAlgo: - description: |- - LBAlgo is the load balancing algorithm. If not specified, defaults to rr. - Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further - details refer to keepalived documentation. - enum: - - rr - - wrr - - lc - - wlc - - lblc - - dh - - sh - - sed - - nq - type: string - lbKind: - description: |- - LBKind is the load balancing kind. If not specified, defaults to DR. - Valid values are NAT DR TUN. For further details refer to keepalived documentation. - enum: - - NAT - - DR - - TUN - type: string - persistenceTimeout: - description: |- - PersistenceTimeout specify a timeout value for persistent connections in - seconds. If not specified, defaults to 360 (6 minutes). - type: integer - type: object - type: array - vrrpInstances: + Keepalived contains configuration options related to the "Keepalived" type + of load balancing. + properties: + virtualServers: + description: |- + Configuration options related to the virtual servers. This is an array + which allows to configure multiple load balancers. + items: + description: VirtualServer defines the configuration + options for a virtual server. + properties: + delayLoop: + description: DelayLoop is the delay timer for check + polling. If not specified, defaults to 0. + type: integer + ipAddress: + description: IPAddress is the virtual IP address + used by the virtual server. + type: string + lbAlgo: + description: |- + LBAlgo is the load balancing algorithm. If not specified, defaults to rr. + Valid values are rr, wrr, lc, wlc, lblc, dh, sh, sed, nq. For further + details refer to keepalived documentation. + enum: + - rr + - wrr + - lc + - wlc + - lblc + - dh + - sh + - sed + - nq + type: string + lbKind: + description: |- + LBKind is the load balancing kind. If not specified, defaults to DR. + Valid values are NAT DR TUN. For further details refer to keepalived documentation. + enum: + - NAT + - DR + - TUN + type: string + persistenceTimeout: + description: |- + PersistenceTimeout specify a timeout value for persistent connections in + seconds. If not specified, defaults to 360 (6 minutes). + type: integer + type: object + type: array + vrrpInstances: + description: |- + Configuration options related to the VRRP. This is an array which allows + to configure multiple virtual IPs. + items: + description: VRRPInstance defines the configuration + options for a VRRP instance. + properties: + advertInterval: + default: 1 + description: |- + AdvertInterval is the advertisement interval in seconds. If not specified, + use 1 second + format: int32 + type: integer + authPass: + description: |- + AuthPass is the password for accessing vrrpd. This is not a security + feature but a way to prevent accidental misconfigurations. + Authpass must be 8 characters or less. + type: string + interface: + description: |- + Interface specifies the NIC used by the virtual router. If not specified, + k0s will use the interface that owns the default route. + type: string + name: + default: k0s-vip + description: |- + Name is the name of the VRRP instance. If not specified, defaults to + k0s-vip-. + type: string + virtualIPs: + description: |- + VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs + must be a CIDR as defined in RFC 4632 and RFC 4291. + items: + type: string + type: array + virtualRouterID: + default: 51 + description: |- + VirtualRouterID is the VRRP router ID. If not specified, defaults to 51. + VirtualRouterID must be in the range of 1-255, all the control plane + nodes must have the same VirtualRouterID. + Two clusters in the same network must not use the same VirtualRouterID. + format: int32 + maximum: 255 + minimum: 1 + type: integer + type: object + type: array + type: object + type: + default: Keepalived description: |- - Configuration options related to the VRRP. This is an array which allows - to configure multiple virtual IPs. - items: - description: VRRPInstance defines the configuration options - for a VRRP instance. - properties: - advertInterval: - default: 1 - description: |- - AdvertInterval is the advertisement interval in seconds. If not specified, - use 1 second - format: int32 - type: integer - authPass: - description: |- - AuthPass is the password for accessing vrrpd. This is not a security - feature but a way to prevent accidental misconfigurations. - Authpass must be 8 characters or less. - type: string - interface: - description: |- - Interface specifies the NIC used by the virtual router. If not specified, - k0s will use the interface that owns the default route. - type: string - name: - default: k0s-vip - description: |- - Name is the name of the VRRP instance. If not specified, defaults to - k0s-vip-. - type: string - virtualIPs: - description: |- - VirtualIP is the list virtual IP address used by the VRRP instance. VirtualIPs - must be a CIDR as defined in RFC 4632 and RFC 4291. - items: - type: string - type: array - virtualRouterID: - default: 51 - description: |- - VirtualRouterID is the VRRP router ID. If not specified, defaults to 51. - VirtualRouterID must be in the range of 1-255, all the control plane - nodes must have the same VirtualRouterID. - Two clusters in the same network must not use the same VirtualRouterID. - format: int32 - maximum: 255 - minimum: 1 - type: integer - type: object - type: array + type indicates the type of the node-local load balancer to deploy on + worker nodes. Currently, the only supported type is "Keepalived". + enum: + - keepalived + type: string type: object dualStack: description: DualStack defines network configuration for ipv4\ipv6