E2E tests - [DO NOT MERGE]

Mirantis · Nov 18, 2024 · fb94528 · fb94528
1 parent a21cdec
commit fb94528
Show file tree

Hide file tree

Showing 17 changed files with 249 additions and 70 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -1,6 +1,6 @@
 name: CI
 on:
-  pull_request_target:
+  pull_request:
     types:
       - labeled
       - opened
@@ -23,29 +23,14 @@ env:
 jobs:
   build:
     concurrency:
-      group: build-${{ github.head_ref || github.run_id }}
+      group: ${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     name: Build and Unit Test
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.vars.outputs.version }}
-      clustername: ${{ steps.vars.outputs.clustername }}
-      pr: ${{ steps.pr.outputs.result }}
     steps:
-      - name: Get PR ref
-        uses: actions/github-script@v7
-        id: pr
-        with:
-          script: |
-            const { data: pullRequest } = await github.rest.pulls.get({
-              ...context.repo,
-              pull_number: context.payload.pull_request.number,
-            });
-            return pullRequest
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          ref: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}}
           fetch-depth: 0
       - name: Setup Go
         uses: actions/setup-go@v5
@@ -59,6 +44,31 @@ jobs:
       - name: Unit tests
         run: |
           make test
+      - name: Build HMC controller image
+        run: |
+          make docker-build
+
+  push:
+    concurrency:
+      group: push-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    name: E2E Push Images and Charts to GHCR
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      clustername: ${{ steps.vars.outputs.clustername }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
       - name: Set up Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to GHCR
@@ -73,7 +83,8 @@ jobs:
           GIT_VERSION=$(git describe --tags --always)
           echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT
           echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
-      - name: Build and push HMC controller image
+      - name: Push HMC Controller Image to GHCR
+        if:
         uses: docker/build-push-action@v6
         with:
           build-args: |
@@ -85,36 +96,36 @@ jobs:
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
-      - name: Prepare and push HMC template charts
+      - name: Prepare and push HMC template charts to GHCR
         run: |
+          make set-hmc-version
           make hmc-chart-release
           make helm-push
 
   controller-e2etest:
     name: E2E Controller
     runs-on: ubuntu-latest
-    needs: build
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
+    needs: push
     concurrency:
       group: controller-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup kubectl
         uses: azure/setup-kubectl@v4
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'controller'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -129,14 +140,13 @@ jobs:
     name: E2E Cloud Providers
     runs-on: ubuntu-latest
     if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
-    needs: build
+    needs: push
     concurrency:
       group: cloud-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     env:
       AWS_REGION: us-west-2
       AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
@@ -151,7 +161,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -162,9 +171,9 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:cloud'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -179,14 +188,13 @@ jobs:
     name: E2E On-Prem Providers
     runs-on: self-hosted
     if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
-    needs: build
+    needs: push
     concurrency:
       group: onprem-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     env:
       VSPHERE_USER: ${{ secrets.CI_VSPHERE_USER }}
       VSPHERE_PASSWORD: ${{ secrets.CI_VSPHERE_PASSWORD }}
@@ -205,7 +213,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -215,9 +222,9 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:onprem'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -229,23 +236,21 @@ jobs:
             test/e2e/*.log
 
   cleanup:
-    name: Cleanup
+    name: E2E Cleanup
     needs:
-      - build
+      - push
       - provider-cloud-e2etest
     runs-on: ubuntu-latest
-    if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }}
+    if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.push.result, 'success') }}
     timeout-minutes: 15
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -260,7 +265,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
-          CLUSTER_NAME: '${{ needs.build.outputs.clustername }}'
+          CLUSTER_NAME: '${{ needs.push.outputs.clustername }}'
         run: |
           make dev-aws-nuke
-          make dev-azure-nuke
+          make dev-azure-nuke
diff --git a/templates/provider/hmc/templates/rbac/controller/roles.yaml b/templates/provider/hmc/templates/rbac/controller/roles.yaml
@@ -191,6 +191,11 @@ rules:
   - clusterprofiles
   - clustersummaries
   verbs: {{ include "rbac.editorVerbs" . | nindent 4 }}
+- apiGroups:
+  - controlplane.cluster.x-k8s.io
+  resources:
+  - awsmanagedcontrolplanes
+  verbs: {{ include "rbac.viewerVerbs" . | nindent 4 }}
 - apiGroups:
   - hmc.mirantis.com
   resources:

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -53,7 +53,8 @@ var _ = BeforeSuite(func() {
 	_, err := utils.Run(cmd)
 	Expect(err).NotTo(HaveOccurred())
 	cmd = exec.Command("make", "test-apply")
-	_, err = utils.Run(cmd)
+	output, err := utils.Run(cmd)
+	_, _ = fmt.Fprintln(GinkgoWriter, string(output))
 	Expect(err).NotTo(HaveOccurred())
 
 	By("validating that the hmc-controller and CAPI provider controllers are running and ready")
@@ -217,8 +218,10 @@ func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, provider
 			"describe", "cluster", clusterName, "--namespace", internalutils.DefaultSystemNamespace, "--show-conditions=all")
 		output, err := utils.Run(cmd)
 		if err != nil {
-			utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err))
-			return
+			if !strings.Contains(err.Error(), "unable to verify clusterctl version") {
+				utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err))
+				return
+			}
 		}
 		err = os.WriteFile(filepath.Join("test/e2e", host+"-"+"clusterctl.log"), output, 0o644)
 		if err != nil {

diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -36,6 +37,11 @@ import (
 	"github.com/Mirantis/hmc/internal/utils/status"
 )
 
+const (
+	numOfRetries        = 10
+	initialRetryTimeout = time.Second * 1
+)
+
 type KubeClient struct {
 	Client         kubernetes.Interface
 	ExtendedClient apiextensionsclientset.Interface
@@ -194,9 +200,20 @@ func (kc *KubeClient) CreateManagedCluster(
 		Resource: "managedclusters",
 	}, true)
 
-	_, err := client.Create(ctx, managedcluster, metav1.CreateOptions{})
-	if !apierrors.IsAlreadyExists(err) {
-		Expect(err).NotTo(HaveOccurred(), "failed to create %s", kind)
+	i := 0
+	timeout := initialRetryTimeout
+	for i < numOfRetries {
+		_, err := client.Create(ctx, managedcluster, metav1.CreateOptions{})
+		if err == nil || apierrors.IsAlreadyExists(err) {
+			break
+		}
+		i++
+		if i == numOfRetries {
+			Expect(err).NotTo(HaveOccurred(), "failed to create %s after %d retries", kind, numOfRetries)
+		}
+		_, _ = fmt.Fprintf(GinkgoWriter, "Create ManagedCluster, attempt #%d failed, retrying after %d seconds...\n", i, timeout)
+		time.Sleep(timeout)
+		timeout *= 2
 	}
 
 	return func() error {
@@ -279,3 +296,15 @@ func (kc *KubeClient) ListK0sControlPlanes(
 		Resource: "k0scontrolplanes",
 	}, clusterName)
 }
+
+func (kc *KubeClient) ListAWSManagedControlPlanes(
+	ctx context.Context, clusterName string,
+) ([]unstructured.Unstructured, error) {
+	GinkgoHelper()
+
+	return kc.listResource(ctx, schema.GroupVersionResource{
+		Group:    "controlplane.cluster.x-k8s.io",
+		Version:  "v1beta2",
+		Resource: "awsmanagedcontrolplanes",
+	}, clusterName)
+}
diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go
@@ -50,6 +50,7 @@ const (
 	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
 	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
 	TemplateVSphereHostedCP     Template = "vsphere-hosted-cp"
+	TemplateEKSCP               Template = "aws-eks-cp"
 )
 
 //go:embed resources/aws-standalone-cp.yaml.tpl
@@ -70,6 +71,9 @@ var vsphereStandaloneCPManagedClusterTemplateBytes []byte
 //go:embed resources/vsphere-hosted-cp.yaml.tpl
 var vsphereHostedCPManagedClusterTemplateBytes []byte
 
+//go:embed resources/aws-eks-cp.yaml.tpl
+var eksCPManagedClusterTemplateBytes []byte
+
 func FilterAllProviders() []string {
 	return []string{
 		utils.HMCControllerLabel,
@@ -134,6 +138,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes
 	case TemplateAzureStandaloneCP:
 		managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes
+	case TemplateEKSCP:
+		managedClusterTemplateBytes = eksCPManagedClusterTemplateBytes
 	default:
 		Fail(fmt.Sprintf("Unsupported template: %s", templateName))
 	}

diff --git a/test/e2e/managedcluster/providervalidator.go b/test/e2e/managedcluster/providervalidator.go
@@ -65,14 +65,22 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
+		case TemplateEKSCP:
+			resourcesToValidate["control-planes"] = validateAWSManagedControlPlanes
+			delete(resourcesToValidate, "csi-driver")
 		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
 		}
 	} else {
+		validateCPDeletedFunc := validateK0sControlPlanesDeleted
+		if template == TemplateEKSCP {
+			validateCPDeletedFunc = validateAWSManagedControlPlanesDeleted
+		}
+
 		resourcesToValidate = map[string]resourceValidationFunc{
 			"clusters":           validateClusterDeleted,
 			"machinedeployments": validateMachineDeploymentsDeleted,
-			"control-planes":     validateK0sControlPlanesDeleted,
+			"control-planes":     validateCPDeletedFunc,
 		}
 		resourceOrder = []string{"clusters", "machinedeployments", "control-planes"}
 	}