From 5e872622f8a38a43455c2f278222603cfda63fbc Mon Sep 17 00:00:00 2001 From: Slava Lysunkin Date: Tue, 19 Nov 2024 10:20:50 -0600 Subject: [PATCH] E2E tests with EKS --- .../hmc/templates/rbac/controller/roles.yaml | 5 +++ test/e2e/e2e_suite_test.go | 6 ++- test/e2e/kubeclient/kubeclient.go | 12 ++++++ test/e2e/managedcluster/managedcluster.go | 6 +++ test/e2e/managedcluster/providervalidator.go | 10 ++++- .../resources/aws-eks-cp.yaml.tpl | 16 ++++++++ test/e2e/managedcluster/validate_deleted.go | 18 ++++++++ test/e2e/managedcluster/validate_deployed.go | 28 +++++++++++-- test/e2e/provider_aws_test.go | 41 +++++++++++++++++++ test/utils/utils.go | 27 +++++++++++- 10 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl diff --git a/templates/provider/hmc/templates/rbac/controller/roles.yaml b/templates/provider/hmc/templates/rbac/controller/roles.yaml index d37c4f8f..799a10a6 100644 --- a/templates/provider/hmc/templates/rbac/controller/roles.yaml +++ b/templates/provider/hmc/templates/rbac/controller/roles.yaml @@ -191,6 +191,11 @@ rules: - clusterprofiles - clustersummaries verbs: {{ include "rbac.editorVerbs" . | nindent 4 }} +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes + verbs: {{ include "rbac.viewerVerbs" . | nindent 4 }} - apiGroups: - hmc.mirantis.com resources: diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e76a4c24..48aab8d1 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -217,8 +217,10 @@ func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, provider "describe", "cluster", clusterName, "--namespace", internalutils.DefaultSystemNamespace, "--show-conditions=all") output, err := utils.Run(cmd) if err != nil { - utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err)) - return + if !strings.Contains(err.Error(), "unable to verify clusterctl version") { + utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err)) + return + } } err = os.WriteFile(filepath.Join("test/e2e", host+"-"+"clusterctl.log"), output, 0o644) if err != nil { diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index e3801e4e..d5e328aa 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes( Resource: "k0scontrolplanes", }, clusterName) } + +func (kc *KubeClient) ListAWSManagedControlPlanes( + ctx context.Context, clusterName string, +) ([]unstructured.Unstructured, error) { + GinkgoHelper() + + return kc.listResource(ctx, schema.GroupVersionResource{ + Group: "controlplane.cluster.x-k8s.io", + Version: "v1beta2", + Resource: "awsmanagedcontrolplanes", + }, clusterName) +} diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index 72efa96f..e8be90e1 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -50,6 +50,7 @@ const ( TemplateAzureStandaloneCP Template = "azure-standalone-cp" TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp" TemplateVSphereHostedCP Template = "vsphere-hosted-cp" + TemplateEKSCP Template = "aws-eks-cp" ) //go:embed resources/aws-standalone-cp.yaml.tpl @@ -70,6 +71,9 @@ var vsphereStandaloneCPManagedClusterTemplateBytes []byte //go:embed resources/vsphere-hosted-cp.yaml.tpl var vsphereHostedCPManagedClusterTemplateBytes []byte +//go:embed resources/aws-eks-cp.yaml.tpl +var eksCPManagedClusterTemplateBytes []byte + func FilterAllProviders() []string { return []string{ utils.HMCControllerLabel, @@ -134,6 +138,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes case TemplateAzureStandaloneCP: managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes + case TemplateEKSCP: + managedClusterTemplateBytes = eksCPManagedClusterTemplateBytes default: Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } diff --git a/test/e2e/managedcluster/providervalidator.go b/test/e2e/managedcluster/providervalidator.go index 4df0fa84..3ccb5a07 100644 --- a/test/e2e/managedcluster/providervalidator.go +++ b/test/e2e/managedcluster/providervalidator.go @@ -65,14 +65,22 @@ func NewProviderValidator(template Template, clusterName string, action Validati case TemplateAWSStandaloneCP, TemplateAWSHostedCP: resourcesToValidate["ccm"] = validateCCM resourceOrder = append(resourceOrder, "ccm") + case TemplateEKSCP: + resourcesToValidate["control-planes"] = validateAWSManagedControlPlanes + delete(resourcesToValidate, "csi-driver") case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP: delete(resourcesToValidate, "csi-driver") } } else { + validateCPDeletedFunc := validateK0sControlPlanesDeleted + if template == TemplateEKSCP { + validateCPDeletedFunc = validateAWSManagedControlPlanesDeleted + } + resourcesToValidate = map[string]resourceValidationFunc{ "clusters": validateClusterDeleted, "machinedeployments": validateMachineDeploymentsDeleted, - "control-planes": validateK0sControlPlanesDeleted, + "control-planes": validateCPDeletedFunc, } resourceOrder = []string{"clusters", "machinedeployments", "control-planes"} } diff --git a/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl new file mode 100644 index 00000000..1b0cc50a --- /dev/null +++ b/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl @@ -0,0 +1,16 @@ +apiVersion: hmc.mirantis.com/v1alpha1 +kind: ManagedCluster +metadata: + name: ${MANAGED_CLUSTER_NAME}-eks +spec: + template: aws-eks-0-0-2 + credential: ${AWS_CLUSTER_IDENTITY}-cred + config: + region: ${AWS_REGION} + workersNumber: ${WORKERS_NUMBER:=1} + clusterIdentity: + name: ${AWS_CLUSTER_IDENTITY}-cred + namespace: ${NAMESPACE} + publicIP: ${AWS_PUBLIC_IP:=true} + worker: + instanceType: ${AWS_INSTANCE_TYPE:=t3.small} diff --git a/test/e2e/managedcluster/validate_deleted.go b/test/e2e/managedcluster/validate_deleted.go index e09d4c25..4272f9fe 100644 --- a/test/e2e/managedcluster/validate_deleted.go +++ b/test/e2e/managedcluster/validate_deleted.go @@ -101,3 +101,21 @@ func validateK0sControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeCli return nil } + +func validateAWSManagedControlPlanesDeleted(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + + var cpNames []string + if len(controlPlanes) > 0 { + for _, cp := range controlPlanes { + cpNames = append(cpNames, cp.GetName()) + + return fmt.Errorf("AWS Managed control planes still exist: %s", cpNames) + } + } + + return nil +} diff --git a/test/e2e/managedcluster/validate_deployed.go b/test/e2e/managedcluster/validate_deployed.go index bae823f7..7a600ff3 100644 --- a/test/e2e/managedcluster/validate_deployed.go +++ b/test/e2e/managedcluster/validate_deployed.go @@ -55,7 +55,7 @@ func validateCluster(ctx context.Context, kc *kubeclient.KubeClient, clusterName Fail(err.Error()) } - return utils.ValidateConditionsTrue(cluster) + return utils.NewConditionsValidator().IfTrue(cluster) } func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { @@ -79,7 +79,7 @@ func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterNam Fail(err.Error()) } - if err := utils.ValidateConditionsTrue(&md); err != nil { + if err := utils.NewConditionsValidator().IfTrue(&md); err != nil { return err } } @@ -90,7 +90,7 @@ func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterNam Fail(err.Error()) } - if err := utils.ValidateConditionsTrue(&machine); err != nil { + if err := utils.NewConditionsValidator().IfTrue(&machine); err != nil { return err } } @@ -113,7 +113,7 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl // k0s does not use the metav1.Condition type for status.conditions, // instead it uses a custom type so we can't use - // ValidateConditionsTrue here, instead we'll check for "ready: true". + // ordinary conditions validation here, instead we'll check for "ready: true". objStatus, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status") if !found { return fmt.Errorf("no status found for %s: %s", objKind, objName) @@ -139,6 +139,26 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl return nil } +func validateAWSManagedControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { + controlPlanes, err := kc.ListAWSManagedControlPlanes(ctx, clusterName) + if err != nil { + return err + } + + for _, controlPlane := range controlPlanes { + if err := utils.ValidateObjectNamePrefix(&controlPlane, clusterName); err != nil { + Fail(err.Error()) + } + + // EKSControlPlaneCreating condition very often has READY=False, SEVERITY=Info and REASON=created (this is fine). + if err := utils.NewConditionsValidator(utils.WithExcluded([]string{"EKSControlPlaneCreating"})).IfTrue(&controlPlane); err != nil { + return err + } + } + + return nil +} + // validateCSIDriver validates that the provider CSI driver is functioning // by creating a PVC and verifying it enters "Bound" status. func validateCSIDriver(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error { diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 6614698b..c7e651fa 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -186,4 +186,45 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order time.Second).Should(Succeed()) */ }) + + It("should work with an EKS provider", func() { + // Deploy a standalone cluster and verify it is running/ready. + GinkgoT().Setenv(managedcluster.EnvVarAWSInstanceType, "t3.small") + + cmd := exec.Command("kubectl", "get", "clustertemplates", "-n", "hmc-system", "-o", "yaml") + output, err := utils.Run(cmd) + _, _ = fmt.Fprintln(GinkgoWriter, string(output)) + Expect(err).NotTo(HaveOccurred()) + + templateBy(managedcluster.TemplateEKSCP, "creating a ManagedCluster for EKS") + sd := managedcluster.GetUnstructured(managedcluster.TemplateEKSCP) + clusterName = sd.GetName() + + standaloneDeleteFunc = kc.CreateManagedCluster(context.Background(), sd) + + templateBy(managedcluster.TemplateEKSCP, "waiting for infrastructure to deploy successfully") + deploymentValidator := managedcluster.NewProviderValidator( + managedcluster.TemplateEKSCP, + clusterName, + managedcluster.ValidationActionDeploy, + ) + + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(60 * time.Minute).WithPolling(30 * time.Second).Should(Succeed()) + + // --- clean up --- + templateBy(managedcluster.TemplateAWSStandaloneCP, "deleting the ManagedCluster for EKS") + Expect(standaloneDeleteFunc()).NotTo(HaveOccurred()) + + deletionValidator := managedcluster.NewProviderValidator( + managedcluster.TemplateAWSStandaloneCP, + clusterName, + managedcluster.ValidationActionDelete, + ) + Eventually(func() error { + return deletionValidator.Validate(context.Background(), kc) + }).WithTimeout(15 * time.Minute).WithPolling(10 * + time.Second).Should(Succeed()) + }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index 56ce5ee3..79212558 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/strings/slices" "github.com/Mirantis/hmc/internal/utils/status" ) @@ -111,10 +112,28 @@ func GetProjectDir() (string, error) { return wd, nil } -// ValidateConditionsTrue iterates over the conditions of the given +type ConditionsValidator struct { + excludedConditions []string +} + +func NewConditionsValidator(options ...func(*ConditionsValidator)) *ConditionsValidator { + cv := &ConditionsValidator{} + for _, o := range options { + o(cv) + } + return cv +} + +func WithExcluded(excludedConditions []string) func(*ConditionsValidator) { + return func(cv *ConditionsValidator) { + cv.excludedConditions = excludedConditions + } +} + +// IfTrue iterates over the conditions of the given // unstructured object and returns an error if any of the conditions are not // true. Conditions are expected to be of type metav1.Condition. -func ValidateConditionsTrue(unstrObj *unstructured.Unstructured) error { +func (cv *ConditionsValidator) IfTrue(unstrObj *unstructured.Unstructured) error { objKind, objName := status.ObjKindName(unstrObj) conditions, err := status.ConditionsFromUnstructured(unstrObj) @@ -129,6 +148,10 @@ func ValidateConditionsTrue(unstrObj *unstructured.Unstructured) error { continue } + if slices.Contains(cv.excludedConditions, c.Type) { + continue + } + errs = errors.Join(errors.New(ConvertConditionsToString(c)), errs) }