Use statically initialized runtime schemes

This fixes a potential panic during k0s startup when components are started at the same time that register new types for shared schemes. Signed-off-by: Tom Wieczorek <[email protected]> (cherry picked from commit 7743df2)
k0sproject · Jun 5, 2024 · d8a5e6d · d8a5e6d
1 parent f704a85
commit d8a5e6d
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 17 deletions.
diff --git a/pkg/autopilot/controller/root_controller.go b/pkg/autopilot/controller/root_controller.go
@@ -27,7 +27,6 @@ import (
 	aproot "github.com/k0sproject/k0s/pkg/autopilot/controller/root"
 	apsig "github.com/k0sproject/k0s/pkg/autopilot/controller/signal"
 	apupdate "github.com/k0sproject/k0s/pkg/autopilot/controller/updates"
-	apscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme"
 	"github.com/k0sproject/k0s/pkg/kubernetes"
 
 	"github.com/sirupsen/logrus"
@@ -148,6 +147,7 @@ func (c *rootController) Run(ctx context.Context) error {
 // and starts it in a goroutine.
 func (c *rootController) startSubControllerRoutine(ctx context.Context, logger *logrus.Entry, event LeaseEventStatus) error {
 	managerOpts := crman.Options{
+		Scheme: scheme,
 		WebhookServer: crwebhook.NewServer(crwebhook.Options{
 			Port: c.cfg.ManagerPort,
 		}),
@@ -163,11 +163,6 @@ func (c *rootController) startSubControllerRoutine(ctx context.Context, logger *
 		return err
 	}
 
-	if err := apscheme.AddToScheme(mgr.GetScheme()); err != nil {
-		logger.WithError(err).Error("unable to register autopilot scheme")
-		return err
-	}
-
 	if err := RegisterIndexers(ctx, mgr, "controller"); err != nil {
 		logger.WithError(err).Error("unable to register indexers")
 		return err

diff --git a/pkg/autopilot/controller/root_worker.go b/pkg/autopilot/controller/root_worker.go
@@ -23,7 +23,6 @@ import (
 	apdel "github.com/k0sproject/k0s/pkg/autopilot/controller/delegate"
 	aproot "github.com/k0sproject/k0s/pkg/autopilot/controller/root"
 	apsig "github.com/k0sproject/k0s/pkg/autopilot/controller/signal"
-	apscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme"
 
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -60,6 +59,7 @@ func (w *rootWorker) Run(ctx context.Context) error {
 	logger := w.log
 
 	managerOpts := crman.Options{
+		Scheme: scheme,
 		WebhookServer: crwebhook.NewServer(crwebhook.Options{
 			Port: w.cfg.ManagerPort,
 		}),
@@ -85,10 +85,6 @@ func (w *rootWorker) Run(ctx context.Context) error {
 		logger.WithError(err).Fatal("unable to start controller manager")
 	}
 
-	if err := apscheme.AddToScheme(mgr.GetScheme()); err != nil {
-		logger.WithError(err).Fatal("unable to register autopilot scheme")
-	}
-
 	// In some cases, we need to wait on the worker side until controller deploys all autopilot CRDs
 	return k8sretry.OnError(wait.Backoff{
 		Steps:    120,

diff --git a/pkg/autopilot/controller/scheme.go b/pkg/autopilot/controller/scheme.go
@@ -0,0 +1,31 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	k0sscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	k8sscheme "k8s.io/client-go/kubernetes/scheme"
+)
+
+var scheme = runtime.NewScheme()
+
+func init() {
+	utilruntime.Must(k8sscheme.AddToScheme(scheme))
+	utilruntime.Must(k0sscheme.AddToScheme(scheme))
+}
diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go
@@ -31,6 +31,7 @@ import (
 	helmapi "github.com/k0sproject/k0s/pkg/apis/helm"
 	"github.com/k0sproject/k0s/pkg/apis/helm/v1beta1"
 	k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	k0sscheme "github.com/k0sproject/k0s/pkg/client/clientset/scheme"
 	"github.com/k0sproject/k0s/pkg/component/controller/leaderelector"
 	"github.com/k0sproject/k0s/pkg/component/manager"
 	"github.com/k0sproject/k0s/pkg/config"
@@ -46,6 +47,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	apiretry "k8s.io/client-go/util/retry"
+	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config"
@@ -417,7 +419,8 @@ func (ec *ExtensionsController) Start(ctx context.Context) error {
 		Kind:  "Chart",
 	}
 
-	mgr, err := crman.New(clientConfig, crman.Options{
+	mgr, err := controllerruntime.NewManager(clientConfig, crman.Options{
+		Scheme: k0sscheme.Scheme,
 		Metrics: metricsserver.Options{
 			BindAddress: "0",
 		},
@@ -438,11 +441,6 @@ func (ec *ExtensionsController) Start(ctx context.Context) error {
 	}, retry.Context(ctx)); err != nil {
 		return fmt.Errorf("can't start ExtensionsReconciler, helm CRD is not registred, check CRD registration reconciler: %w", err)
 	}
-	// examples say to not use GetScheme in production, but it is unclear at the moment
-	// which scheme should be in use
-	if err := v1beta1.AddToScheme(mgr.GetScheme()); err != nil {
-		return fmt.Errorf("can't register Chart crd: %w", err)
-	}
 
 	if err := builder.
 		ControllerManagedBy(mgr).