From 7743df2c3fc18427709f41e15edbd2bc7ea6c967 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Wed, 5 Jun 2024 13:15:39 +0200 Subject: [PATCH] Use statically initialized runtime schemes This fixes a potential panic during k0s startup when components are started at the same time that register new types for shared schemes. Signed-off-by: Tom Wieczorek --- pkg/autopilot/controller/root_controller.go | 7 +---- pkg/autopilot/controller/root_worker.go | 6 +--- pkg/autopilot/controller/scheme.go | 31 +++++++++++++++++++ .../controller/extensions_controller.go | 10 +++--- 4 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 pkg/autopilot/controller/scheme.go diff --git a/pkg/autopilot/controller/root_controller.go b/pkg/autopilot/controller/root_controller.go index cc5b0f4bffc8..6102b32636b8 100644 --- a/pkg/autopilot/controller/root_controller.go +++ b/pkg/autopilot/controller/root_controller.go @@ -27,7 +27,6 @@ import ( aproot "github.com/k0sproject/k0s/pkg/autopilot/controller/root" apsig "github.com/k0sproject/k0s/pkg/autopilot/controller/signal" apupdate "github.com/k0sproject/k0s/pkg/autopilot/controller/updates" - apscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme" "github.com/k0sproject/k0s/pkg/kubernetes" "github.com/sirupsen/logrus" @@ -148,6 +147,7 @@ func (c *rootController) Run(ctx context.Context) error { // and starts it in a goroutine. func (c *rootController) startSubControllerRoutine(ctx context.Context, logger *logrus.Entry, event LeaseEventStatus) error { managerOpts := crman.Options{ + Scheme: scheme, WebhookServer: crwebhook.NewServer(crwebhook.Options{ Port: c.cfg.ManagerPort, }), @@ -163,11 +163,6 @@ func (c *rootController) startSubControllerRoutine(ctx context.Context, logger * return err } - if err := apscheme.AddToScheme(mgr.GetScheme()); err != nil { - logger.WithError(err).Error("unable to register autopilot scheme") - return err - } - if err := RegisterIndexers(ctx, mgr, "controller"); err != nil { logger.WithError(err).Error("unable to register indexers") return err diff --git a/pkg/autopilot/controller/root_worker.go b/pkg/autopilot/controller/root_worker.go index 2a3e635ab315..abc15fd1020b 100644 --- a/pkg/autopilot/controller/root_worker.go +++ b/pkg/autopilot/controller/root_worker.go @@ -23,7 +23,6 @@ import ( apdel "github.com/k0sproject/k0s/pkg/autopilot/controller/delegate" aproot "github.com/k0sproject/k0s/pkg/autopilot/controller/root" apsig "github.com/k0sproject/k0s/pkg/autopilot/controller/signal" - apscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -60,6 +59,7 @@ func (w *rootWorker) Run(ctx context.Context) error { logger := w.log managerOpts := crman.Options{ + Scheme: scheme, WebhookServer: crwebhook.NewServer(crwebhook.Options{ Port: w.cfg.ManagerPort, }), @@ -85,10 +85,6 @@ func (w *rootWorker) Run(ctx context.Context) error { logger.WithError(err).Fatal("unable to start controller manager") } - if err := apscheme.AddToScheme(mgr.GetScheme()); err != nil { - logger.WithError(err).Fatal("unable to register autopilot scheme") - } - // In some cases, we need to wait on the worker side until controller deploys all autopilot CRDs return k8sretry.OnError(wait.Backoff{ Steps: 120, diff --git a/pkg/autopilot/controller/scheme.go b/pkg/autopilot/controller/scheme.go new file mode 100644 index 000000000000..54a851725d75 --- /dev/null +++ b/pkg/autopilot/controller/scheme.go @@ -0,0 +1,31 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + k0sscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + k8sscheme "k8s.io/client-go/kubernetes/scheme" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(k0sscheme.AddToScheme(scheme)) +} diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index e507247ce8da..5f7778de94de 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -31,6 +31,7 @@ import ( helmapi "github.com/k0sproject/k0s/pkg/apis/helm" "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + k0sscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme" "github.com/k0sproject/k0s/pkg/component/controller/leaderelector" "github.com/k0sproject/k0s/pkg/component/manager" "github.com/k0sproject/k0s/pkg/config" @@ -46,6 +47,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" apiretry "k8s.io/client-go/util/retry" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" @@ -417,7 +419,8 @@ func (ec *ExtensionsController) Start(ctx context.Context) error { Kind: "Chart", } - mgr, err := crman.New(clientConfig, crman.Options{ + mgr, err := controllerruntime.NewManager(clientConfig, crman.Options{ + Scheme: k0sscheme.Scheme, Metrics: metricsserver.Options{ BindAddress: "0", }, @@ -438,11 +441,6 @@ func (ec *ExtensionsController) Start(ctx context.Context) error { }, retry.Context(ctx)); err != nil { return fmt.Errorf("can't start ExtensionsReconciler, helm CRD is not registred, check CRD registration reconciler: %w", err) } - // examples say to not use GetScheme in production, but it is unclear at the moment - // which scheme should be in use - if err := v1beta1.AddToScheme(mgr.GetScheme()); err != nil { - return fmt.Errorf("can't register Chart crd: %w", err) - } if err := builder. ControllerManagedBy(mgr).