From 8e5f7e2a2ae8c6340f046cce0c504763e1005b7e Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Fri, 12 Jul 2024 17:43:07 +0200 Subject: [PATCH 1/4] Use the helm clientset as the extension controller's scheme Not that the helm clientset has been backported, its static scheme can be used instead of the ad-hoc created one. See: 0b2b3ffd6 ("Use helm.k0sproject.io/v1beta1 for the extensions controller's scheme") Signed-off-by: Tom Wieczorek --- pkg/component/controller/extensions_controller.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index 93a24db18818..4bbe56262183 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -26,6 +26,7 @@ import ( "github.com/bombsimon/logrusr/v2" "github.com/k0sproject/k0s/internal/pkg/templatewriter" "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" + helmscheme "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1/clientset/scheme" k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/k0sproject/k0s/pkg/component/controller/leaderelector" "github.com/k0sproject/k0s/pkg/component/manager" @@ -35,7 +36,6 @@ import ( "github.com/sirupsen/logrus" "helm.sh/helm/v3/pkg/release" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" @@ -372,13 +372,8 @@ func (ec *ExtensionsController) Start(ctx context.Context) error { Kind: "Chart", } - scheme := runtime.NewScheme() - if err := v1beta1.AddToScheme(scheme); err != nil { - return err - } - mgr, err := controllerruntime.NewManager(clientConfig, ctrlManager.Options{ - Scheme: scheme, + Scheme: helmscheme.Scheme, MetricsBindAddress: "0", Logger: logrusr.New(ec.L), Controller: config.Controller{}, From 96db3515344b544dc5cf07a76f2d754bf60c16b6 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Tue, 2 Jul 2024 15:04:19 +0200 Subject: [PATCH 2/4] Collect errors during reconciliaion in extension controller This way, the reconciliation gets as close as possible to the desired state, even if some things in between are failing. Signed-off-by: Tom Wieczorek (cherry picked from commit e0aa6a4977041b18f303d60c3dac7b330122fe22) (cherry picked from commit 9db7b1e8eb2a7a093c7a42326822f3feb02c5b05) (cherry picked from commit 0b638de8d89d247bd8dd4b669638ce441919901a) (cherry picked from commit 550baf0d2de50814bbae291894f7de629fcac227) --- .../controller/extensions_controller.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index 4bbe56262183..ee23a059d030 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -19,6 +19,7 @@ package controller import ( "bytes" "context" + "errors" "fmt" "time" @@ -35,7 +36,7 @@ import ( kubeutil "github.com/k0sproject/k0s/pkg/kubernetes" "github.com/sirupsen/logrus" "helm.sh/helm/v3/pkg/release" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" @@ -152,9 +153,10 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExt return nil } + var errs []error for _, repo := range helmSpec.Repositories { if err := ec.addRepo(repo); err != nil { - return fmt.Errorf("can't init repository %q: %w", repo.URL, err) + errs = append(errs, fmt.Errorf("can't init repository %q: %w", repo.URL, err)) } } @@ -172,13 +174,16 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExt } buf := bytes.NewBuffer([]byte{}) if err := tw.WriteToBuffer(buf); err != nil { - return fmt.Errorf("can't create chart CR instance %q: %w", chart.ChartName, err) + errs = append(errs, fmt.Errorf("can't create chart CR instance %q: %w", chart.ChartName, err)) + continue } if err := ec.saver.Save(chart.ManifestFileName(), buf.Bytes()); err != nil { - return fmt.Errorf("can't save addon CRD manifest for chart CR instance %q: %w", chart.ChartName, err) + errs = append(errs, fmt.Errorf("can't save addon CRD manifest for chart CR instance %q: %w", chart.ChartName, err)) + continue } } - return nil + + return errors.Join(errs...) } type ChartReconciler struct { @@ -198,7 +203,7 @@ func (cr *ChartReconciler) Reconcile(ctx context.Context, req reconcile.Request) var chartInstance v1beta1.Chart if err := cr.Client.Get(ctx, req.NamespacedName, &chartInstance); err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return reconcile.Result{}, nil } return reconcile.Result{}, err From 4f801749e94fa098ec427ae1ddee2ae56882d1bc Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Tue, 2 Jul 2024 15:05:26 +0200 Subject: [PATCH 3/4] Move chart file name generation to extensions controller This is an internal implementation detail of the extensions controller and not an inherent property of the chart resource. Hence move the func into the controller and make it private. Signed-off-by: Tom Wieczorek (cherry picked from commit 394418d848de24ddfdd03f7f574d4e15f6ff1791) (cherry picked from commit 966dc5ec0b9658c04dab2f66f44a17e374ba4100) (cherry picked from commit 0cbc962890431790412207d93551104de1b8afff) (cherry picked from commit 9c438e1d4577c7ecb8252ad2dc2544eef305d24c) --- .../k0s.k0sproject.io/v1beta1/extensions.go | 6 --- .../v1beta1/extenstions_test.go | 26 ---------- .../controller/extensions_controller.go | 49 ++++++++++--------- .../controller/extensions_controller_test.go | 27 ++++++++++ 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/extensions.go b/pkg/apis/k0s.k0sproject.io/v1beta1/extensions.go index f65b9f94053f..d7d04ee8cfe5 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/extensions.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/extensions.go @@ -18,7 +18,6 @@ package v1beta1 import ( "errors" - "fmt" "time" "helm.sh/helm/v3/pkg/chartutil" @@ -99,11 +98,6 @@ type Chart struct { Order int `json:"order"` } -// ManifestFileName returns filename to use for the crd manifest -func (c Chart) ManifestFileName() string { - return fmt.Sprintf("%d_helm_extension_%s.yaml", c.Order, c.Name) -} - // Validate performs validation func (c Chart) Validate() error { if c.Name == "" { diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/extenstions_test.go b/pkg/apis/k0s.k0sproject.io/v1beta1/extenstions_test.go index 752c773e22bc..b2abbe6a8634 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/extenstions_test.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/extenstions_test.go @@ -81,30 +81,4 @@ func TestValidation(t *testing.T) { }) }) - - t.Run("chart_manifest_name", func(t *testing.T) { - chart := Chart{ - Name: "release", - ChartName: "k0s/chart", - TargetNS: "default", - } - - chart1 := Chart{ - Name: "release", - ChartName: "k0s/chart", - TargetNS: "default", - Order: 1, - } - - chart2 := Chart{ - Name: "release", - ChartName: "k0s/chart", - TargetNS: "default", - Order: 2, - } - assert.Equal(t, chart.ManifestFileName(), "0_helm_extension_release.yaml") - assert.Equal(t, chart1.ManifestFileName(), "1_helm_extension_release.yaml") - assert.Equal(t, chart2.ManifestFileName(), "2_helm_extension_release.yaml") - }) - } diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index ee23a059d030..9b35ae46b9b7 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -26,9 +26,9 @@ import ( "github.com/avast/retry-go" "github.com/bombsimon/logrusr/v2" "github.com/k0sproject/k0s/internal/pkg/templatewriter" - "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" + helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" helmscheme "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1/clientset/scheme" - k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/k0sproject/k0s/pkg/component/controller/leaderelector" "github.com/k0sproject/k0s/pkg/component/manager" "github.com/k0sproject/k0s/pkg/constant" @@ -80,14 +80,14 @@ const ( ) // Run runs the extensions controller -func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0sAPI.ClusterConfig) error { +func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0sv1beta1.ClusterConfig) error { ec.L.Info("Extensions reconciliation started") defer ec.L.Info("Extensions reconciliation finished") helmSettings := clusterConfig.Spec.Extensions.Helm var err error switch clusterConfig.Spec.Extensions.Storage.Type { - case k0sAPI.OpenEBSLocal: + case k0sv1beta1.OpenEBSLocal: helmSettings, err = addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage) if err != nil { ec.L.WithError(err).Error("Can't add openebs helm extension") @@ -102,7 +102,7 @@ func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0 return nil } -func addOpenEBSHelmExtension(helmSpec *k0sAPI.HelmExtensions, storageExtension *k0sAPI.StorageExtension) (*k0sAPI.HelmExtensions, error) { +func addOpenEBSHelmExtension(helmSpec *k0sv1beta1.HelmExtensions, storageExtension *k0sv1beta1.StorageExtension) (*k0sv1beta1.HelmExtensions, error) { openEBSValues := map[string]interface{}{ "localprovisioner": map[string]interface{}{ "hostpathClass": map[string]interface{}{ @@ -117,16 +117,16 @@ func addOpenEBSHelmExtension(helmSpec *k0sAPI.HelmExtensions, storageExtension * return nil, err } if helmSpec == nil { - helmSpec = &k0sAPI.HelmExtensions{ - Repositories: k0sAPI.RepositoriesSettings{}, - Charts: k0sAPI.ChartsSettings{}, + helmSpec = &k0sv1beta1.HelmExtensions{ + Repositories: k0sv1beta1.RepositoriesSettings{}, + Charts: k0sv1beta1.ChartsSettings{}, } } - helmSpec.Repositories = append(helmSpec.Repositories, k0sAPI.Repository{ + helmSpec.Repositories = append(helmSpec.Repositories, k0sv1beta1.Repository{ Name: "openebs-internal", URL: constant.OpenEBSRepository, }) - helmSpec.Charts = append(helmSpec.Charts, k0sAPI.Chart{ + helmSpec.Charts = append(helmSpec.Charts, k0sv1beta1.Chart{ Name: "openebs", ChartName: "openebs-internal/openebs", TargetNS: "openebs", @@ -148,7 +148,7 @@ func yamlifyValues(values map[string]interface{}) (string, error) { // reconcileHelmExtensions creates instance of Chart CR for each chart of the config file // it also reconciles repositories settings // the actual helm install/update/delete management is done by ChartReconciler structure -func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExtensions) error { +func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sv1beta1.HelmExtensions) error { if helmSpec == nil { return nil } @@ -165,7 +165,7 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExt Name: "addon_crd_manifest", Template: chartCrdTemplate, Data: struct { - k0sAPI.Chart + k0sv1beta1.Chart Finalizer string }{ Chart: chart, @@ -177,7 +177,7 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExt errs = append(errs, fmt.Errorf("can't create chart CR instance %q: %w", chart.ChartName, err)) continue } - if err := ec.saver.Save(chart.ManifestFileName(), buf.Bytes()); err != nil { + if err := ec.saver.Save(chartManifestFileName(&chart), buf.Bytes()); err != nil { errs = append(errs, fmt.Errorf("can't save addon CRD manifest for chart CR instance %q: %w", chart.ChartName, err)) continue } @@ -186,6 +186,11 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sAPI.HelmExt return errors.Join(errs...) } +// Determines the file name to use when storing a chart as a manifest on disk. +func chartManifestFileName(c *k0sv1beta1.Chart) string { + return fmt.Sprintf("%d_helm_extension_%s.yaml", c.Order, c.Name) +} + type ChartReconciler struct { client.Client helm *helm.Commands @@ -200,7 +205,7 @@ func (cr *ChartReconciler) Reconcile(ctx context.Context, req reconcile.Request) cr.L.Tracef("Got helm chart reconciliation request: %s", req) defer cr.L.Tracef("Finished processing helm chart reconciliation request: %s", req) - var chartInstance v1beta1.Chart + var chartInstance helmv1beta1.Chart if err := cr.Client.Get(ctx, req.NamespacedName, &chartInstance); err != nil { if apierrors.IsNotFound(err) { @@ -229,7 +234,7 @@ func (cr *ChartReconciler) Reconcile(ctx context.Context, req reconcile.Request) cr.L.Debugf("Installed or updated reconciliation request: %s", req) return reconcile.Result{}, nil } -func (cr *ChartReconciler) uninstall(ctx context.Context, chart v1beta1.Chart) error { +func (cr *ChartReconciler) uninstall(ctx context.Context, chart helmv1beta1.Chart) error { if err := cr.helm.UninstallRelease(ctx, chart.Status.ReleaseName, chart.Status.Namespace); err != nil { return fmt.Errorf("can't uninstall release `%s/%s`: %w", chart.Status.Namespace, chart.Status.ReleaseName, err) } @@ -238,7 +243,7 @@ func (cr *ChartReconciler) uninstall(ctx context.Context, chart v1beta1.Chart) e const defaultTimeout = time.Duration(10 * time.Minute) -func (cr *ChartReconciler) updateOrInstallChart(ctx context.Context, chart v1beta1.Chart) error { +func (cr *ChartReconciler) updateOrInstallChart(ctx context.Context, chart helmv1beta1.Chart) error { var err error var chartRelease *release.Release timeout, err := time.ParseDuration(chart.Spec.Timeout) @@ -298,7 +303,7 @@ func (cr *ChartReconciler) updateOrInstallChart(ctx context.Context, chart v1bet return nil } -func (cr *ChartReconciler) chartNeedsUpgrade(chart v1beta1.Chart) bool { +func (cr *ChartReconciler) chartNeedsUpgrade(chart helmv1beta1.Chart) bool { return !(chart.Status.Namespace == chart.Spec.Namespace && chart.Status.ReleaseName == chart.Spec.ReleaseName && chart.Status.Version == chart.Spec.Version && @@ -310,9 +315,9 @@ func (cr *ChartReconciler) chartNeedsUpgrade(chart v1beta1.Chart) bool { // to complete and the chart may have been updated in the meantime. If returns the error returned // by the Update operation. Moreover, if the chart has indeed changed in the meantime we already // have an event for it so we will see it again soon. -func (cr *ChartReconciler) updateStatus(ctx context.Context, chart v1beta1.Chart, chartRelease *release.Release, err error) error { +func (cr *ChartReconciler) updateStatus(ctx context.Context, chart helmv1beta1.Chart, chartRelease *release.Release, err error) error { nsn := types.NamespacedName{Namespace: chart.Namespace, Name: chart.Name} - var updchart v1beta1.Chart + var updchart helmv1beta1.Chart if err := cr.Get(ctx, nsn, &updchart); err != nil { return fmt.Errorf("can't get updated version of chart %s: %w", chart.Name, err) } @@ -337,7 +342,7 @@ func (cr *ChartReconciler) updateStatus(ctx context.Context, chart v1beta1.Chart return nil } -func (ec *ExtensionsController) addRepo(repo k0sAPI.Repository) error { +func (ec *ExtensionsController) addRepo(repo k0sv1beta1.Repository) error { return ec.helm.AddRepository(repo) } @@ -373,7 +378,7 @@ func (ec *ExtensionsController) Start(ctx context.Context) error { return fmt.Errorf("can't build controller-runtime controller for helm extensions: %w", err) } gk := schema.GroupKind{ - Group: v1beta1.SchemeGroupVersion.Group, + Group: helmv1beta1.SchemeGroupVersion.Group, Kind: "Chart", } @@ -400,7 +405,7 @@ func (ec *ExtensionsController) Start(ctx context.Context) error { if err := builder. ControllerManagedBy(mgr). - For(&v1beta1.Chart{}, + For(&helmv1beta1.Chart{}, builder.WithPredicates(predicate.And( predicate.GenerationChangedPredicate{}, predicate.NewPredicateFuncs(func(object client.Object) bool { diff --git a/pkg/component/controller/extensions_controller_test.go b/pkg/component/controller/extensions_controller_test.go index 8b1264e25c5e..3a15bb89467b 100644 --- a/pkg/component/controller/extensions_controller_test.go +++ b/pkg/component/controller/extensions_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/stretchr/testify/assert" ) @@ -134,3 +135,29 @@ func TestChartNeedsUpgrade(t *testing.T) { }) } } + +func TestChartManifestFileName(t *testing.T) { + chart := k0sv1beta1.Chart{ + Name: "release", + ChartName: "k0s/chart", + TargetNS: "default", + } + + chart1 := k0sv1beta1.Chart{ + Name: "release", + ChartName: "k0s/chart", + TargetNS: "default", + Order: 1, + } + + chart2 := k0sv1beta1.Chart{ + Name: "release", + ChartName: "k0s/chart", + TargetNS: "default", + Order: 2, + } + + assert.Equal(t, chartManifestFileName(&chart), "0_helm_extension_release.yaml") + assert.Equal(t, chartManifestFileName(&chart1), "1_helm_extension_release.yaml") + assert.Equal(t, chartManifestFileName(&chart2), "2_helm_extension_release.yaml") +} From e785828024d8faab738a110bc7bd86bcdca9c8b6 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Tue, 2 Jul 2024 15:13:13 +0200 Subject: [PATCH 4/4] Cleanup unknown Helm chart manifest files Keep track of the generated filenames when reconciling Helm Chart extensions. After all files have been synchronized, remove any remaining unknown Helm chart manifest files. This way the synchronization will work correctly even if the Helm chart extension names and orders are changed. Signed-off-by: Tom Wieczorek (cherry picked from commit c1f8c7533a3bbaaa934620b14ff7da56fbed3f8f) (cherry picked from commit 0e4db6a3b3db821884070448b10d99448632a443) (cherry picked from commit bfaff3ba9c6883a43ff86a57333ec5eb07f9ad7c) (cherry picked from commit 6dc0306d495fd5095b41469d8d1034a44922229e) --- cmd/controller/controller.go | 1 - inttest/addons/addons_test.go | 70 ++++++++++++++++--- .../controller/extensions_controller.go | 54 +++++++++++--- .../controller/extensions_controller_test.go | 1 + 4 files changed, 106 insertions(+), 20 deletions(-) diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 1430e16e4502..4dc7dd8fbf4e 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -356,7 +356,6 @@ func (c *command) start(ctx context.Context) error { } c.ClusterComponents.Add(ctx, controller.NewCRD(helmSaver, []string{"helm"})) c.ClusterComponents.Add(ctx, controller.NewExtensionsController( - helmSaver, c.K0sVars, adminClientFactory, leaderElector, diff --git a/inttest/addons/addons_test.go b/inttest/addons/addons_test.go index e24b3f02243b..c3c81439ffd0 100644 --- a/inttest/addons/addons_test.go +++ b/inttest/addons/addons_test.go @@ -18,7 +18,9 @@ package addons import ( "bytes" + "context" "fmt" + "slices" "testing" "time" @@ -26,8 +28,12 @@ import ( "github.com/k0sproject/k0s/internal/pkg/templatewriter" "github.com/k0sproject/k0s/inttest/common" - "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1" + helmclientset "github.com/k0sproject/k0s/pkg/apis/helm.k0sproject.io/v1beta1/clientset" + k0sclientset "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/constant" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" k8s "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -40,12 +46,14 @@ type AddonsSuite struct { } func (as *AddonsSuite) TestHelmBasedAddons() { + ctx := as.Context() + addonName := "test-addon" ociAddonName := "oci-addon" fileAddonName := "tgz-addon" as.PutFile(as.ControllerNode(0), "/tmp/k0s.yaml", fmt.Sprintf(k0sConfigWithAddon, addonName)) as.pullHelmChart(as.ControllerNode(0)) - as.Require().NoError(as.InitController(0, "--config=/tmp/k0s.yaml")) + as.Require().NoError(as.InitController(0, "--config=/tmp/k0s.yaml", "--enable-dynamic-config")) as.NoError(as.RunWorkers()) kc, err := as.KubeClient(as.ControllerNode(0)) as.Require().NoError(err) @@ -57,6 +65,8 @@ func (as *AddonsSuite) TestHelmBasedAddons() { as.AssertSomeKubeSystemPods(kc) + as.Run("Rename chart in Helm extension", func() { as.renameChart(ctx) }) + values := map[string]interface{}{ "replicaCount": 2, "image": map[string]interface{}{ @@ -83,7 +93,49 @@ func (as *AddonsSuite) pullHelmChart(node string) { as.Require().NoError(err) } -func (as *AddonsSuite) deleteRelease(chart *v1beta1.Chart) { +func (as *AddonsSuite) renameChart(ctx context.Context) { + restConfig, err := as.GetKubeConfig(as.ControllerNode(0)) + as.Require().NoError(err) + k0sClients, err := k0sclientset.NewForConfig(restConfig) + as.Require().NoError(err) + helmClients, err := helmclientset.NewForConfig(restConfig) + as.Require().NoError(err) + + configs := k0sClients.K0sV1beta1().ClusterConfigs(constant.ClusterConfigNamespace) + cfg, err := configs.Get(ctx, constant.ClusterConfigObjectName, metav1.GetOptions{}) + as.Require().NoError(err) + + i := slices.IndexFunc(cfg.Spec.Extensions.Helm.Charts, func(c k0sv1beta1.Chart) bool { + return c.Name == "tgz-addon" + }) + as.Require().GreaterOrEqual(i, 0, "Didn't find tgz-addon in %v", cfg.Spec.Extensions.Helm.Charts) + cfg.Spec.Extensions.Helm.Charts[i].Name = "tgz-renamed-addon" + + cfg, err = configs.Update(ctx, cfg, metav1.UpdateOptions{FieldManager: as.T().Name()}) + as.Require().NoError(err) + if data, err := yaml.Marshal(cfg); as.NoError(err) { + as.T().Logf("%s", data) + } + + as.Require().NoError(wait.PollUntilContextCancel(ctx, 350*time.Millisecond, true, func(ctx context.Context) (bool, error) { + charts, err := helmClients.HelmV1beta1().Charts(constant.ClusterConfigNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, nil + } + + hasChart := func(name string) bool { + return slices.IndexFunc(charts.Items, func(c helmv1beta1.Chart) bool { + return c.Name == name + }) >= 0 + } + + return !hasChart("k0s-addon-chart-tgz-addon") && hasChart("k0s-addon-chart-tgz-renamed-addon"), nil + }), "While waiting for Chart resource to be swapped") + + as.waitForTestRelease("tgz-renamed-addon", "0.6.0", "kube-system", 1) +} + +func (as *AddonsSuite) deleteRelease(chart *helmv1beta1.Chart) { as.T().Logf("Deleting chart %s/%s", chart.Namespace, chart.Name) ssh, err := as.SSH(as.Context(), as.ControllerNode(0)) as.Require().NoError(err) @@ -96,7 +148,7 @@ func (as *AddonsSuite) deleteRelease(chart *v1beta1.Chart) { as.Require().NoError(err) as.Require().NoError(wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) { as.T().Logf("Expecting have no secrets left for release %s/%s", chart.Namespace, chart.Name) - items, err := k8sclient.CoreV1().Secrets("default").List(as.Context(), v1.ListOptions{ + items, err := k8sclient.CoreV1().Secrets("default").List(as.Context(), metav1.ListOptions{ LabelSelector: fmt.Sprintf("name=%s", chart.Name), }) if err != nil { @@ -111,18 +163,18 @@ func (as *AddonsSuite) deleteRelease(chart *v1beta1.Chart) { })) } -func (as *AddonsSuite) waitForTestRelease(addonName, appVersion string, namespace string, rev int64) *v1beta1.Chart { +func (as *AddonsSuite) waitForTestRelease(addonName, appVersion string, namespace string, rev int64) *helmv1beta1.Chart { as.T().Logf("waiting to see %s release ready in kube API, generation %d", addonName, rev) cfg, err := as.GetKubeConfig(as.ControllerNode(0)) as.Require().NoError(err) - err = v1beta1.AddToScheme(scheme.Scheme) + err = helmv1beta1.AddToScheme(scheme.Scheme) as.Require().NoError(err) chartClient, err := client.New(cfg, client.Options{ Scheme: scheme.Scheme, }) as.Require().NoError(err) - var chart v1beta1.Chart + var chart helmv1beta1.Chart as.Require().NoError(wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) { err = chartClient.Get(as.Context(), client.ObjectKey{ Namespace: "kube-system", @@ -163,7 +215,7 @@ func (as *AddonsSuite) checkCustomValues(releaseName string) error { } return wait.PollImmediate(time.Second, 2*time.Minute, func() (done bool, err error) { serverDeployment := fmt.Sprintf("%s-echo-server", releaseName) - d, err := kc.AppsV1().Deployments("default").Get(as.Context(), serverDeployment, v1.GetOptions{}) + d, err := kc.AppsV1().Deployments("default").Get(as.Context(), serverDeployment, metav1.GetOptions{}) if err != nil { return false, nil } diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index 9b35ae46b9b7..8d46896e5dbe 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -17,10 +17,14 @@ limitations under the License. package controller import ( - "bytes" "context" "errors" "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" "time" "github.com/avast/retry-go" @@ -54,24 +58,24 @@ import ( // Helm watch for Chart crd type ExtensionsController struct { - saver manifestsSaver L *logrus.Entry helm *helm.Commands kubeConfig string leaderElector leaderelector.Interface + manifestsDir string } var _ manager.Component = (*ExtensionsController)(nil) var _ manager.Reconciler = (*ExtensionsController)(nil) // NewExtensionsController builds new HelmAddons -func NewExtensionsController(s manifestsSaver, k0sVars constant.CfgVars, kubeClientFactory kubeutil.ClientFactoryInterface, leaderElector leaderelector.Interface) *ExtensionsController { +func NewExtensionsController(k0sVars constant.CfgVars, kubeClientFactory kubeutil.ClientFactoryInterface, leaderElector leaderelector.Interface) *ExtensionsController { return &ExtensionsController{ - saver: s, L: logrus.WithFields(logrus.Fields{"component": "extensions_controller"}), helm: helm.NewCommands(k0sVars), kubeConfig: k0sVars.AdminKubeConfigPath, leaderElector: leaderElector, + manifestsDir: filepath.Join(k0sVars.ManifestsDir, "helm"), } } @@ -160,8 +164,13 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sv1beta1.Hel } } + var fileNamesToKeep []string for _, chart := range helmSpec.Charts { + fileName := chartManifestFileName(&chart) + fileNamesToKeep = append(fileNamesToKeep, fileName) + tw := templatewriter.TemplateWriter{ + Path: filepath.Join(ec.manifestsDir, fileName), Name: "addon_crd_manifest", Template: chartCrdTemplate, Data: struct { @@ -172,15 +181,35 @@ func (ec *ExtensionsController) reconcileHelmExtensions(helmSpec *k0sv1beta1.Hel Finalizer: finalizerName, }, } - buf := bytes.NewBuffer([]byte{}) - if err := tw.WriteToBuffer(buf); err != nil { - errs = append(errs, fmt.Errorf("can't create chart CR instance %q: %w", chart.ChartName, err)) + if err := tw.Write(); err != nil { + errs = append(errs, fmt.Errorf("can't write file for Helm chart manifest %q: %w", chart.ChartName, err)) continue } - if err := ec.saver.Save(chartManifestFileName(&chart), buf.Bytes()); err != nil { - errs = append(errs, fmt.Errorf("can't save addon CRD manifest for chart CR instance %q: %w", chart.ChartName, err)) - continue + + ec.L.Infof("Wrote Helm chart manifest file %q", tw.Path) + } + + if err := filepath.WalkDir(ec.manifestsDir, func(path string, entry fs.DirEntry, err error) error { + switch { + case !entry.Type().IsRegular(): + ec.L.Debugf("Keeping %v as it is not a regular file", entry) + case slices.Contains(fileNamesToKeep, entry.Name()): + ec.L.Debugf("Keeping %v as it belongs to a known Helm extension", entry) + case !isChartManifestFileName(entry.Name()): + ec.L.Debugf("Keeping %v as it is not a Helm chart manifest file", entry) + default: + if err := os.Remove(path); err != nil { + if !errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("failed to remove Helm chart manifest file, the Chart resource will remain in the cluster: %w", err)) + } + } else { + ec.L.Infof("Removed Helm chart manifest file %q", path) + } } + + return nil + }); err != nil { + errs = append(errs, fmt.Errorf("failed to walk Helm chart manifest directory: %w", err)) } return errors.Join(errs...) @@ -191,6 +220,11 @@ func chartManifestFileName(c *k0sv1beta1.Chart) string { return fmt.Sprintf("%d_helm_extension_%s.yaml", c.Order, c.Name) } +// Determines if the given file name is in the format for chart manifest file names. +func isChartManifestFileName(fileName string) bool { + return regexp.MustCompile(`^-?[0-9]+_helm_extension_.+\.yaml$`).MatchString(fileName) +} + type ChartReconciler struct { client.Client helm *helm.Commands diff --git a/pkg/component/controller/extensions_controller_test.go b/pkg/component/controller/extensions_controller_test.go index 3a15bb89467b..fbbb7e3a52e0 100644 --- a/pkg/component/controller/extensions_controller_test.go +++ b/pkg/component/controller/extensions_controller_test.go @@ -160,4 +160,5 @@ func TestChartManifestFileName(t *testing.T) { assert.Equal(t, chartManifestFileName(&chart), "0_helm_extension_release.yaml") assert.Equal(t, chartManifestFileName(&chart1), "1_helm_extension_release.yaml") assert.Equal(t, chartManifestFileName(&chart2), "2_helm_extension_release.yaml") + assert.True(t, isChartManifestFileName("0_helm_extension_release.yaml")) }