Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow CPLB to work with externalAddress #4452

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions cmd/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,24 +228,26 @@ func (c *command) start(ctx context.Context) error {
nodeComponents.Add(ctx, controllerLeaseCounter)
}

disableEndpointReconciler := !slices.Contains(c.DisableComponents, constant.APIEndpointReconcilerComponentName) &&
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This condition is also used at line 436/438 for the endpoint reconciler component itself. Would be cool to reuse this boolean there as well. And it could deserve a less ambiguous name: Basically it swaps endpoint reconcilers. What about naming this preferExternalAddressEndpointReconciler, preferK0sEndpointReconciler or sth. like that?

nodeConfig.Spec.API.ExternalAddress != ""

if cplb := nodeConfig.Spec.Network.ControlPlaneLoadBalancing; cplb != nil && cplb.Enabled {
if c.SingleNode {
return errors.New("control plane load balancing cannot be used in a single-node cluster")
}

nodeComponents.Add(ctx, &controller.Keepalived{
K0sVars: c.K0sVars,
Config: cplb.Keepalived,
DetailedLogging: c.Debug,
LogConfig: c.Debug,
KubeConfigPath: c.K0sVars.AdminKubeConfigPath,
APIPort: nodeConfig.Spec.API.Port,
K0sVars: c.K0sVars,
Config: cplb.Keepalived,
DetailedLogging: c.Debug,
LogConfig: c.Debug,
KubeConfigPath: c.K0sVars.AdminKubeConfigPath,
APIPort: nodeConfig.Spec.API.Port,
HasEndpointReconciler: disableEndpointReconciler,
})
}

enableKonnectivity := !c.SingleNode && !slices.Contains(c.DisableComponents, constant.KonnectivityServerComponentName)
disableEndpointReconciler := !slices.Contains(c.DisableComponents, constant.APIEndpointReconcilerComponentName) &&
nodeConfig.Spec.API.ExternalAddress != ""

if enableKonnectivity {
nodeComponents.Add(ctx, &controller.Konnectivity{
Expand Down
297 changes: 272 additions & 25 deletions docs/cplb.md

Large diffs are not rendered by default.

9 changes: 4 additions & 5 deletions inttest/cplb/cplb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type keepalivedSuite struct {

const haControllerConfig = `
spec:
api:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we test both setups here? One with externalAddress and one with NLLB?

externalAddress: %s
network:
controlPlaneLoadBalancing:
enabled: true
Expand All @@ -43,9 +45,6 @@ spec:
authPass: "123456"
virtualServers:
- ipAddress: %s
nodeLocalLoadBalancing:
enabled: true
type: EnvoyProxy
`

// SetupTest prepares the controller and filesystem, getting it into a consistent
Expand All @@ -57,10 +56,10 @@ func (s *keepalivedSuite) TestK0sGetsUp() {

for idx := 0; idx < s.BootlooseSuite.ControllerCount; idx++ {
s.Require().NoError(s.WaitForSSH(s.ControllerNode(idx), 2*time.Minute, 1*time.Second))
s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb))
s.PutFile(s.ControllerNode(idx), "/tmp/k0s.yaml", fmt.Sprintf(haControllerConfig, lb, lb, lb))

// Note that the token is intentionally empty for the first controller
s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server", joinToken))
s.Require().NoError(s.InitController(idx, "--config=/tmp/k0s.yaml", "--disable-components=metrics-server,endpoint-reconciler", joinToken))
s.Require().NoError(s.WaitJoinAPI(s.ControllerNode(idx)))

// With the primary controller running, create the join token for subsequent controllers.
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/k0s/v1beta1/clusterconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ func (s *ClusterSpec) Validate() (errs []error) {
}

if s.Network != nil && s.Network.ControlPlaneLoadBalancing != nil {
for _, err := range s.Network.ControlPlaneLoadBalancing.Validate(s.API.ExternalAddress) {
for _, err := range s.Network.ControlPlaneLoadBalancing.Validate() {
errs = append(errs, fmt.Errorf("controlPlaneLoadBalancing: %w", err))
}
}
Expand Down
11 changes: 3 additions & 8 deletions pkg/apis/k0s/v1beta1/cplb.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ func (k *KeepalivedSpec) validateVirtualServers() []error {
}

// Validate validates the ControlPlaneLoadBalancingSpec
func (c *ControlPlaneLoadBalancingSpec) Validate(externalAddress string) (errs []error) {
func (c *ControlPlaneLoadBalancingSpec) Validate() (errs []error) {
if c == nil {
return nil
}
Expand All @@ -293,21 +293,16 @@ func (c *ControlPlaneLoadBalancingSpec) Validate(externalAddress string) (errs [
errs = append(errs, fmt.Errorf("unsupported CPLB type: %s. Only allowed value: %s", c.Type, CPLBTypeKeepalived))
}

return append(errs, c.Keepalived.Validate(externalAddress)...)
return append(errs, c.Keepalived.Validate()...)
}

// Validate validates the KeepalivedSpec
func (k *KeepalivedSpec) Validate(externalAddress string) (errs []error) {
func (k *KeepalivedSpec) Validate() (errs []error) {
if k == nil {
return nil
}

errs = append(errs, k.validateVRRPInstances(nil)...)
errs = append(errs, k.validateVirtualServers()...)
// CPLB reconciler relies in watching kubernetes.default.svc endpoints
if externalAddress != "" && len(k.VirtualServers) > 0 {
errs = append(errs, errors.New(".spec.api.externalAddress and virtual servers cannot be used together"))
}

return errs
}
32 changes: 18 additions & 14 deletions pkg/component/controller/cplb_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,20 +45,21 @@ import (

// Keepalived is the controller for the keepalived process in the control plane load balancing
type Keepalived struct {
K0sVars *config.CfgVars
Config *k0sAPI.KeepalivedSpec
DetailedLogging bool
LogConfig bool
APIPort int
KubeConfigPath string
keepalivedConfig *keepalivedConfig
uid int
supervisor *supervisor.Supervisor
log *logrus.Entry
configFilePath string
reconciler *CPLBReconciler
updateCh chan struct{}
reconcilerDone chan struct{}
K0sVars *config.CfgVars
Config *k0sAPI.KeepalivedSpec
DetailedLogging bool
LogConfig bool
APIPort int
KubeConfigPath string
HasEndpointReconciler bool
keepalivedConfig *keepalivedConfig
uid int
supervisor *supervisor.Supervisor
log *logrus.Entry
configFilePath string
reconciler *CPLBReconciler
updateCh chan struct{}
reconcilerDone chan struct{}
}

// Init extracts the needed binaries and creates the directories
Expand Down Expand Up @@ -92,6 +93,9 @@ func (k *Keepalived) Start(_ context.Context) error {
}

if len(k.Config.VirtualServers) > 0 {
if k.HasEndpointReconciler {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about moving that check out the component into the controller command? Otherwise the HasEndpointReconciler flag is just some "I will fail to start" flag.

return errors.New("virtual servers are not supported with the endpoint-reconciler enabled.")
}
k.log.Info("Starting CPLB reconciler")
updateCh := make(chan struct{}, 1)
k.reconciler = NewCPLBReconciler(k.KubeConfigPath, updateCh)
Expand Down
13 changes: 7 additions & 6 deletions pkg/component/controller/cplb_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,13 @@ import (
// Keepalived doesn't work on windows, so we cannot implement it at all.
// Just create the interface so that the CI doesn't complain.
type Keepalived struct {
K0sVars *config.CfgVars
Config *k0sAPI.KeepalivedSpec
DetailedLogging bool
LogConfig bool
APIPort int
KubeConfigPath string
K0sVars *config.CfgVars
Config *k0sAPI.KeepalivedSpec
DetailedLogging bool
LogConfig bool
APIPort int
KubeConfigPath string
HasEndpointReconciler bool
}

func (k *Keepalived) Init(_ context.Context) error {
Expand Down
Loading