From 82819cb069ed52fe62b269fee9415742f528566c Mon Sep 17 00:00:00 2001 From: Satyam Bhardwaj Date: Wed, 20 Nov 2024 15:22:42 +0530 Subject: [PATCH 1/3] update dev/config aws cluster template version to 0-0-3 Signed-off-by: Satyam Bhardwaj --- README.md | 4 ++-- config/dev/aws-managedcluster.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dbec1b91..a0d35d65 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ spec: iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: "" workersNumber: 2 - template: aws-standalone-cp-0-0-2 + template: aws-standalone-cp-0-0-3 credential: aws-credential dryRun: true ``` @@ -235,7 +235,7 @@ metadata: name: aws-standalone namespace: hmc-system spec: - template: aws-standalone-cp-0-0-2 + template: aws-standalone-cp-0-0-3 credential: aws-credential config: region: us-east-2 diff --git a/config/dev/aws-managedcluster.yaml b/config/dev/aws-managedcluster.yaml index 7bd6707b..0458923d 100644 --- a/config/dev/aws-managedcluster.yaml +++ b/config/dev/aws-managedcluster.yaml @@ -4,7 +4,7 @@ metadata: name: aws-dev namespace: ${NAMESPACE} spec: - template: aws-standalone-cp-0-0-2 + template: aws-standalone-cp-0-0-3 credential: aws-cluster-identity-cred config: controlPlane: From aba50e54072306748252ea09664b23b67006724d Mon Sep 17 00:00:00 2001 From: Andrei Pavlov Date: Wed, 20 Nov 2024 17:23:45 +0700 Subject: [PATCH 2/3] Update template for vsphere dev example Signed-off-by: Andrei Pavlov --- config/dev/vsphere-managedcluster.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dev/vsphere-managedcluster.yaml b/config/dev/vsphere-managedcluster.yaml index ca3d4651..48ceae71 100644 --- a/config/dev/vsphere-managedcluster.yaml +++ b/config/dev/vsphere-managedcluster.yaml @@ -4,7 +4,7 @@ metadata: name: vsphere-dev namespace: ${NAMESPACE} spec: - template: vsphere-standalone-cp-0-0-2 + template: vsphere-standalone-cp-0-0-3 credential: vsphere-cluster-identity-cred config: controlPlaneNumber: 1 From 10e2719e0ed6502ff7c1bc69af22e1f186a3b0a1 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Tue, 19 Nov 2024 15:35:38 +0400 Subject: [PATCH 3/3] Fix providers status reporting in management object Closes #581 --- Makefile | 13 ++- go.mod | 1 + go.sum | 2 + internal/controller/management_controller.go | 74 ++++++++++---- .../controller/management_controller_test.go | 98 ++++++++++++++++++- internal/controller/suite_test.go | 3 + 6 files changed, 168 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index ea0f40c1..20d2ab57 100644 --- a/Makefile +++ b/Makefile @@ -409,6 +409,10 @@ SVELTOS_VERSION ?= v$(shell $(YQ) -r '.appVersion' $(PROVIDER_TEMPLATES_DIR)/pro SVELTOS_NAME ?= sveltos SVELTOS_CRD ?= $(EXTERNAL_CRD_DIR)/$(SVELTOS_NAME)-$(SVELTOS_VERSION).yaml +CAPI_OPERATOR_VERSION ?= v$(shell $(YQ) -r '.dependencies.[] | select(.name == "cluster-api-operator") | .version' $(PROVIDER_TEMPLATES_DIR)/hmc/Chart.yaml) +CAPI_OPERATOR_CRD_PREFIX ?= "operator.cluster.x-k8s.io_" +CAPI_OPERATOR_CRDS ?= capi-operator-crds + ## Tool Binaries KUBECTL ?= kubectl CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) @@ -479,8 +483,15 @@ $(SVELTOS_CRD): | yq $(EXTERNAL_CRD_DIR) rm -f $(EXTERNAL_CRD_DIR)/$(SVELTOS_NAME)* curl -s --fail https://raw.githubusercontent.com/projectsveltos/sveltos/$(SVELTOS_VERSION)/manifest/crds/sveltos_crds.yaml > $(SVELTOS_CRD) +$(CAPI_OPERATOR_CRDS): | $(YQ) $(EXTERNAL_CRD_DIR) + rm -f $(EXTERNAL_CRD_DIR)/$(CAPI_OPERATOR_CRD_PREFIX)* + @$(foreach name, \ + addonproviders bootstrapproviders controlplaneproviders coreproviders infrastructureproviders ipamproviders runtimeextensionproviders, \ + curl -s --fail https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-operator/$(CAPI_OPERATOR_VERSION)/config/crd/bases/$(CAPI_OPERATOR_CRD_PREFIX)${name}.yaml \ + > $(EXTERNAL_CRD_DIR)/$(CAPI_OPERATOR_CRD_PREFIX)${name}-$(CAPI_OPERATOR_VERSION).yaml;) + .PHONY: external-crd -external-crd: $(FLUX_HELM_CRD) $(FLUX_SOURCE_CHART_CRD) $(FLUX_SOURCE_REPO_CRD) $(SVELTOS_CRD) +external-crd: $(FLUX_HELM_CRD) $(FLUX_SOURCE_CHART_CRD) $(FLUX_SOURCE_REPO_CRD) $(SVELTOS_CRD) $(CAPI_OPERATOR_CRDS) .PHONY: kind kind: $(KIND) ## Download kind locally if necessary. diff --git a/go.mod b/go.mod index a4321c89..67958e30 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( k8s.io/client-go v0.31.2 k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 sigs.k8s.io/cluster-api v1.8.5 + sigs.k8s.io/cluster-api-operator v0.14.0 sigs.k8s.io/cluster-api-provider-azure v1.17.1 sigs.k8s.io/cluster-api-provider-vsphere v1.11.3 sigs.k8s.io/controller-runtime v0.19.1 diff --git a/go.sum b/go.sum index c37ab146..59de606a 100644 --- a/go.sum +++ b/go.sum @@ -678,6 +678,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/cluster-api v1.8.5 h1:lNA2fPN4fkXEs+oOQlnwxT/4VwRFBpv5kkSoJG8nqBA= sigs.k8s.io/cluster-api v1.8.5/go.mod h1:pXv5LqLxuIbhGIXykyNKiJh+KrLweSBajVHHitPLyoY= +sigs.k8s.io/cluster-api-operator v0.14.0 h1:0QgO6+XGrNNJnNHKBwvQD5v6w+EaH3Z0RL1nL3wpjA4= +sigs.k8s.io/cluster-api-operator v0.14.0/go.mod h1:euShpVN6HyxXas28HkrYxhCPVDW1UV6ljbRBAeCxp8Y= sigs.k8s.io/cluster-api-provider-azure v1.17.1 h1:f1sTGfv6hAN9WrxeawE4pQ2nRhEKb7AJjH6MhU/wAzg= sigs.k8s.io/cluster-api-provider-azure v1.17.1/go.mod h1:16VtsvIpK8qtNHplG2ZHZ74/JKTzOUQIAWWutjnpvEc= sigs.k8s.io/cluster-api-provider-vsphere v1.11.3 h1:ONrHsZgiR3L/W4572mn9xvdVnqE0fNogIJI5TeaQI0I= diff --git a/internal/controller/management_controller.go b/internal/controller/management_controller.go index 88b0dbb5..21210271 100644 --- a/internal/controller/management_controller.go +++ b/internal/controller/management_controller.go @@ -23,7 +23,8 @@ import ( "strings" fluxv2 "github.com/fluxcd/helm-controller/api/v2" - "github.com/fluxcd/pkg/apis/meta" + fluxmeta "github.com/fluxcd/pkg/apis/meta" + fluxconditions "github.com/fluxcd/pkg/runtime/conditions" sourcev1 "github.com/fluxcd/source-controller/api/v1" "helm.sh/helm/v3/pkg/chartutil" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -32,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -118,6 +120,8 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *hmc.Manag components: make(map[string]hmc.ComponentStatus), compatibilityContracts: make(map[string]hmc.CompatibilityContracts), } + + requeue bool ) for _, component := range components { l.V(1).Info("reconciling components", "component", component) @@ -152,12 +156,11 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *hmc.Manag continue } - if component.Template != hmc.CoreHMCName { - if err := r.checkProviderStatus(ctx, component.Template); err != nil { - updateComponentsStatus(statusAccumulator, component, nil, fmt.Sprintf("Failed to check provider status: %s", err)) - errs = errors.Join(errs, err) - continue - } + if err := r.checkProviderStatus(ctx, component); err != nil { + l.Info("Provider is not yet ready", "template", component.Template, "err", err) + requeue = true + updateComponentsStatus(statusAccumulator, component, nil, err.Error()) + continue } updateComponentsStatus(statusAccumulator, component, template, "") @@ -177,6 +180,9 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *hmc.Manag l.Error(errs, "Multiple errors during Management reconciliation") return ctrl.Result{}, errs } + if requeue { + return ctrl.Result{RequeueAfter: DefaultRequeueInterval}, nil + } return ctrl.Result{}, nil } @@ -256,9 +262,31 @@ func (r *ManagementReconciler) ensureTemplateManagement(ctx context.Context, mgm // checkProviderStatus checks the status of a provider associated with a given // ProviderTemplate name. Since there's no way to determine resource Kind from // the given template iterate over all possible provider types. -func (r *ManagementReconciler) checkProviderStatus(ctx context.Context, providerTemplateName string) error { - var errs error +func (r *ManagementReconciler) checkProviderStatus(ctx context.Context, component component) error { + helmReleaseName := component.helmReleaseName + hr := &fluxv2.HelmRelease{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.SystemNamespace, Name: helmReleaseName}, hr) + if err != nil { + return fmt.Errorf("failed to check provider status: %w", err) + } + hrReadyCondition := fluxconditions.Get(hr, fluxmeta.ReadyCondition) + if hrReadyCondition == nil || hrReadyCondition.ObservedGeneration != hr.Generation { + return fmt.Errorf("HelmRelease %s/%s Ready condition is not updated yet", r.SystemNamespace, helmReleaseName) + } + if !fluxconditions.IsReady(hr) { + return fmt.Errorf("HelmRelease %s/%s is not yet ready: %s", r.SystemNamespace, helmReleaseName, hrReadyCondition.Message) + } + + if hr.Status.History.Latest() == nil { + return fmt.Errorf("HelmRelease %s/%s has empty deployment history in the status", r.SystemNamespace, helmReleaseName) + } + + if !component.isCAPIProvider { + return nil + } + var errs error + var providerFound bool for _, resourceType := range []string{ "coreproviders", "infrastructureproviders", @@ -272,29 +300,33 @@ func (r *ManagementReconciler) checkProviderStatus(ctx context.Context, provider } resourceConditions, err := status.GetResourceConditions(ctx, r.SystemNamespace, r.DynamicClient, gvr, - labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: providerTemplateName}).String(), + labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: hr.Status.History.Latest().Name}).String(), ) if err != nil { if errors.As(err, &status.ResourceNotFoundError{}) { // Check the next resource type. continue } - - return fmt.Errorf("failed to get status for template: %s: %w", providerTemplateName, err) + return fmt.Errorf("failed to get conditions from %s: %w", gvr.Resource, err) } - var falseConditionTypes []string + providerFound = true + + var falseConditionMessages []string for _, condition := range resourceConditions.Conditions { if condition.Status != metav1.ConditionTrue { - falseConditionTypes = append(falseConditionTypes, condition.Type) + falseConditionMessages = append(falseConditionMessages, condition.Message) } } - if len(falseConditionTypes) > 0 { - errs = errors.Join(errs, fmt.Errorf("%s: %s is not yet ready, has false conditions: %s", - resourceConditions.Name, resourceConditions.Kind, strings.Join(falseConditionTypes, ", "))) + if len(falseConditionMessages) > 0 { + errs = errors.Join(errs, fmt.Errorf("%s is not yet ready: %s", + resourceConditions.Kind, strings.Join(falseConditionMessages, ", "))) } } + if !providerFound { + return errors.New("waiting for Cluster API Provider objects to be created") + } return errs } @@ -373,8 +405,9 @@ type component struct { helmReleaseName string targetNamespace string // helm release dependencies - dependsOn []meta.NamespacedObjectReference + dependsOn []fluxmeta.NamespacedObjectReference createNamespace bool + isCAPIProvider bool } func applyHMCDefaults(config *apiextensionsv1.JSON) (*apiextensionsv1.JSON, error) { @@ -427,7 +460,7 @@ func getWrappedComponents(ctx context.Context, cl client.Client, mgmt *hmc.Manag capiComp := component{ Component: mgmt.Spec.Core.CAPI, helmReleaseName: hmc.CoreCAPIName, - dependsOn: []meta.NamespacedObjectReference{{Name: hmc.CoreHMCName}}, + dependsOn: []fluxmeta.NamespacedObjectReference{{Name: hmc.CoreHMCName}}, isCAPIProvider: true, } if capiComp.Template == "" { capiComp.Template = release.Spec.CAPI.Template @@ -439,7 +472,7 @@ func getWrappedComponents(ctx context.Context, cl client.Client, mgmt *hmc.Manag for _, p := range mgmt.Spec.Providers { c := component{ Component: p.Component, helmReleaseName: p.Name, - dependsOn: []meta.NamespacedObjectReference{{Name: hmc.CoreCAPIName}}, + dependsOn: []fluxmeta.NamespacedObjectReference{{Name: hmc.CoreCAPIName}}, isCAPIProvider: true, } // Try to find corresponding provider in the Release object if c.Template == "" { @@ -449,6 +482,7 @@ func getWrappedComponents(ctx context.Context, cl client.Client, mgmt *hmc.Manag if p.Name == hmc.ProviderSveltosName { c.targetNamespace = sveltosTargetNamespace c.createNamespace = true + c.isCAPIProvider = false } components = append(components, c) diff --git a/internal/controller/management_controller_test.go b/internal/controller/management_controller_test.go index f4eee869..8f6df261 100644 --- a/internal/controller/management_controller_test.go +++ b/internal/controller/management_controller_test.go @@ -20,6 +20,8 @@ import ( "time" helmcontrollerv2 "github.com/fluxcd/helm-controller/api/v2" + fluxmeta "github.com/fluxcd/pkg/apis/meta" + fluxconditions "github.com/fluxcd/pkg/runtime/conditions" sourcev1 "github.com/fluxcd/source-controller/api/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,6 +29,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + capioperator "sigs.k8s.io/cluster-api-operator/api/v1alpha2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -102,6 +106,20 @@ var _ = Describe("Management Controller", func() { interval = time.Millisecond * 250 ) + coreComponents := map[string]struct { + templateName string + helmReleaseName string + }{ + "hmc": { + templateName: "test-release-hmc", + helmReleaseName: "hmc", + }, + "capi": { + templateName: "test-release-capi", + helmReleaseName: "capi", + }, + } + // NOTE: other tests for some reason are manipulating with the NS globally and interfer with each other, // so try to avoid depending on their implementation ignoring its removal By("Creating the hmc-system namespace") @@ -120,8 +138,8 @@ var _ = Describe("Management Controller", func() { }, Spec: hmcmirantiscomv1alpha1.ReleaseSpec{ Version: "test-version", - HMC: hmcmirantiscomv1alpha1.CoreProviderTemplate{Template: "test-release-hmc"}, - CAPI: hmcmirantiscomv1alpha1.CoreProviderTemplate{Template: "test-release-capi"}, + HMC: hmcmirantiscomv1alpha1.CoreProviderTemplate{Template: coreComponents["hmc"].templateName}, + CAPI: hmcmirantiscomv1alpha1.CoreProviderTemplate{Template: coreComponents["capi"].templateName}, }, } Expect(k8sClient.Create(ctx, release)).To(Succeed()) @@ -242,7 +260,79 @@ var _ = Describe("Management Controller", func() { By("Checking the Management object does not have the removed component in its spec") Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgmt), mgmt)).To(Succeed()) Expect(mgmt.Status.AvailableProviders).To(BeEmpty()) + + By("Checking the Management components status is populated") + Expect(mgmt.Status.Components).To(HaveLen(2)) // required: capi, hmc + Expect(mgmt.Status.Components).To(BeEquivalentTo(map[string]hmcmirantiscomv1alpha1.ComponentStatus{ + hmcmirantiscomv1alpha1.CoreHMCName: { + Success: false, + Template: providerTemplateRequiredComponent, + Error: fmt.Sprintf("HelmRelease %s/%s Ready condition is not updated yet", helmReleaseNamespace, coreComponents["hmc"].helmReleaseName), + }, + hmcmirantiscomv1alpha1.CoreCAPIName: { + Success: false, + Template: providerTemplateRequiredComponent, + Error: fmt.Sprintf("HelmRelease %s/%s Ready condition is not updated yet", helmReleaseNamespace, coreComponents["capi"].helmReleaseName), + }, + })) + + By("Update core HelmReleases with Ready condition") + + for _, coreComponent := range coreComponents { + helmRelease := &helmcontrollerv2.HelmRelease{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: helmReleaseNamespace, + Name: coreComponent.helmReleaseName, + }, helmRelease) + Expect(err).NotTo(HaveOccurred()) + + fluxconditions.Set(helmRelease, &metav1.Condition{ + Type: fluxmeta.ReadyCondition, + Reason: helmcontrollerv2.InstallSucceededReason, + Status: metav1.ConditionTrue, + }) + helmRelease.Status.History = helmcontrollerv2.Snapshots{ + { + Name: coreComponent.helmReleaseName, + FirstDeployed: metav1.Now(), + LastDeployed: metav1.Now(), + }, + } + Expect(k8sClient.Status().Update(ctx, helmRelease)).To(Succeed()) + } + + By("Create Cluster API CoreProvider object") + + coreProvider := &capioperator.CoreProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "capi", + Namespace: utils.DefaultSystemNamespace, + Labels: map[string]string{ + "helm.toolkit.fluxcd.io/name": coreComponents["capi"].helmReleaseName, + }, + }, + } + Expect(k8sClient.Create(ctx, coreProvider)).To(Succeed()) + + coreProvider.Status.Conditions = clusterv1.Conditions{ + { + Type: clusterv1.ReadyCondition, + Status: "True", + LastTransitionTime: metav1.Now(), + }, + } + + Expect(k8sClient.Status().Update(ctx, coreProvider)).To(Succeed()) + + By("Reconciling the Management object again to ensure the components status is updated") + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(mgmt), + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mgmt), mgmt)).To(Succeed()) Expect(mgmt.Status.Components).To(BeEquivalentTo(map[string]hmcmirantiscomv1alpha1.ComponentStatus{ hmcmirantiscomv1alpha1.CoreHMCName: {Success: true, Template: providerTemplateRequiredComponent}, hmcmirantiscomv1alpha1.CoreCAPIName: {Success: true, Template: providerTemplateRequiredComponent}, @@ -265,6 +355,10 @@ var _ = Describe("Management Controller", func() { Eventually(func() bool { return apierrors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(providerTemplateRequired), &hmcmirantiscomv1alpha1.ProviderTemplate{})) }).WithTimeout(timeout).WithPolling(interval).Should(BeTrue()) + + coreProvider.Finalizers = nil + Expect(k8sClient.Update(ctx, coreProvider)).To(Succeed()) + Expect(k8sClient.Delete(ctx, coreProvider)).To(Succeed()) }) }) }) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index e805f260..d35dfe8d 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -35,6 +35,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + capioperator "sigs.k8s.io/cluster-api-operator/api/v1alpha2" utilyaml "sigs.k8s.io/cluster-api/util/yaml" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -118,6 +119,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = sveltosv1beta1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = capioperator.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme