diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml new file mode 100644 index 00000000..698edf89 --- /dev/null +++ b/.github/workflows/test_e2e.yml @@ -0,0 +1,269 @@ +name: CI +on: + pull_request: + types: + - labeled + - opened + - synchronize + - reopened + paths-ignore: + - 'config/**' + - '**.md' + push: + tags: + - '*' + +env: + GO_VERSION: '1.22' + REGISTRY_REPO: 'oci://ghcr.io/mirantis/hmc/charts-ci' + +jobs: + build: + concurrency: + group: build-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + name: Build and Unit Test + runs-on: ubuntu-latest + outputs: + version: ${{ steps.vars.outputs.version }} + clustername: ${{ steps.vars.outputs.clustername }} + pr: ${{ steps.pr.outputs.result }} + steps: + - name: Get PR ref + uses: actions/github-script@v7 + id: pr + with: + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + ...context.repo, + pull_number: context.payload.pull_request.number, + }); + return pullRequest + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}} + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - name: Lint + run: GOLANGCI_LINT_TIMEOUT=10m make lint + - name: Verify all generated pieces are up-to-date + run: make generate-all && git add -N . && git diff --exit-code + - name: Unit tests + run: | + make test + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Get outputs + id: vars + run: | + GIT_VERSION=$(git describe --tags --always) + echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT + echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT + - name: Build and push HMC controller image + uses: docker/build-push-action@v6 + with: + build-args: | + LD_FLAGS=-s -w -X github.com/Mirantis/hmc/internal/build.Version=${{ steps.vars.outputs.version }} + context: . + platforms: linux/amd64 + tags: | + ghcr.io/mirantis/hmc/controller-ci:${{ steps.vars.outputs.version }} + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Prepare and push HMC template charts + run: | + make hmc-chart-release + make helm-push + make FORCE_PUSH=true test-hmc-version helm-push + + controller-e2etest: + name: E2E Controller + runs-on: ubuntu-latest + needs: build + concurrency: + group: controller-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'controller' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cloud-e2etest-logs + path: | + test/e2e/*.log + + provider-cloud-e2etest: + name: E2E Cloud Providers + runs-on: ubuntu-latest + if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }} + needs: build + concurrency: + group: cloud-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 + AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }} + steps: + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true + timeout-minutes: 90 + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - uses: actions/checkout@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'provider:cloud' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cloud-e2etest-logs + path: | + test/e2e/*.log + + provider-onprem-e2etest: + name: E2E On-Prem Providers + runs-on: self-hosted + if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }} + needs: build + concurrency: + group: onprem-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + env: + VSPHERE_USER: ${{ secrets.CI_VSPHERE_USER }} + VSPHERE_PASSWORD: ${{ secrets.CI_VSPHERE_PASSWORD }} + VSPHERE_SERVER: ${{ secrets.CI_VSPHERE_SERVER }} + VSPHERE_THUMBPRINT: ${{ secrets.CI_VSPHERE_THUMBPRINT }} + VSPHERE_DATACENTER: ${{ secrets.CI_VSPHERE_DATACENTER }} + VSPHERE_DATASTORE: ${{ secrets.CI_VSPHERE_DATASTORE }} + VSPHERE_RESOURCEPOOL: ${{ secrets.CI_VSPHERE_RESOURCEPOOL }} + VSPHERE_FOLDER: ${{ secrets.CI_VSPHERE_FOLDER }} + VSPHERE_CONTROL_PLANE_ENDPOINT: ${{ secrets.CI_VSPHERE_CONTROL_PLANE_ENDPOINT }} + VSPHERE_VM_TEMPLATE: ${{ secrets.CI_VSPHERE_VM_TEMPLATE }} + VSPHERE_NETWORK: ${{ secrets.CI_VSPHERE_NETWORK }} + VSPHERE_SSH_KEY: ${{ secrets.CI_VSPHERE_SSH_KEY }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'provider:onprem' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: onprem-e2etest-logs + path: | + test/e2e/*.log + + cleanup: + name: Cleanup + needs: + - build + - provider-cloud-e2etest + runs-on: ubuntu-latest + if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }} + timeout-minutes: 15 + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: AWS Test Resources + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 + AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }} + CLUSTER_NAME: '${{ needs.build.outputs.clustername }}' + run: | + make dev-aws-nuke + make dev-azure-nuke diff --git a/Makefile b/Makefile index 561d85d3..316361a4 100644 --- a/Makefile +++ b/Makefile @@ -72,8 +72,33 @@ set-hmc-version: yq $(YQ) eval '.metadata.name = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml $(YQ) eval '.spec.hmc.template = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml +.PHONY: test-hmc-version +test-hmc-version: yq + @TEMPLATE_DIR=templates/provider/hmc-templates/files/templates/*; \ + cutversion=$$(echo $(VERSION) | sed "s/.*-//g"); \ + echo "cutversion = $cutversion"; \ + for template in $$TEMPLATE_DIR; do \ + if [ $$($(YQ) eval '.kind' $$template) = 'ClusterTemplate' ]; then \ + if grep --quiet "chartVersion:.*-"$$cutversion $$template; then \ + echo "skipping already processed templates $$template"; \ + else \ + echo $$template; \ + sed -i "s/chartVersion:.*[^$$cutversion].*/&-$$cutversion/g" "$$template"; \ + fi \ + fi; \ + done; \ + CHART_DIR=templates/cluster/**/Chart.yaml; \ + for chart in $$CHART_DIR; do \ + if grep --quiet "version:.*-$$cutversion" $$chart; then \ + echo "skipping already processed templates $$chart"; \ + else \ + echo $$chart; \ + sed -i "s/version:.*[^$$cutversion]*./&-$$cutversion/g" "$$chart"; \ + fi \ + done + .PHONY: hmc-chart-release -hmc-chart-release: set-hmc-version templates-generate ## Generate hmc helm chart +hmc-chart-release: set-hmc-version test-hmc-version templates-generate ## Generate hmc helm chart .PHONY: hmc-dist-release hmc-dist-release: $(HELM) $(YQ) @@ -110,7 +135,7 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling # compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster. +.PHONY: test-e2e test-hmc-version# Run the e2e tests using a Kind k8s instance as the management cluster. test-e2e: cli-install @if [ "$$GINKGO_LABEL_FILTER" ]; then \ ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \ @@ -297,7 +322,7 @@ helm-push: helm-package else \ chart_exists=$$($(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \ fi; \ - if [ -z "$$chart_exists" ]; then \ + if [ -z "$$chart_exists" ] && [ -z $$FORCE_PUSH ]; then \ echo "Chart $$chart_name version $$chart_version already exists in the repository."; \ else \ if $(REGISTRY_IS_OCI); then \ @@ -352,7 +377,7 @@ dev-eks-creds: dev-aws-creds dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-release .PHONY: test-apply -test-apply: set-hmc-version helm-package dev-deploy dev-templates dev-release +test-apply: set-hmc-version test-hmc-version helm-package dev-deploy dev-templates dev-release .PHONY: dev-destroy dev-destroy: kind-undeploy registry-undeploy ## Destroy the development environment by deleting the kind cluster and local registry. diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index 3ec9a7a2..8b94133d 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -19,9 +19,15 @@ spec: - name: {{ .Values.network.nodeSubnetName }} role: node routeTable: - name: {{ .Values.network.routeTableName }} + name: {{ .Values.network.nodeRouteTableName }} securityGroup: - name: {{ .Values.network.securityGroupName }} + name: {{ .Values.network.nodeSecurityGroupName }} + - name: {{ .Values.network.cpSubnetName }} + role: control-plane + routeTable: + name: {{ .Values.network.cpRouteTableName }} + securityGroup: + name: {{ .Values.network.cpSecurityGroupName }} location: {{ .Values.location }} {{- if .Values.bastion.enabled }} {{- with .Values.bastion.bastionSpec }} diff --git a/templates/cluster/azure-hosted-cp/values.schema.json b/templates/cluster/azure-hosted-cp/values.schema.json index dd04d037..75e3947f 100644 --- a/templates/cluster/azure-hosted-cp/values.schema.json +++ b/templates/cluster/azure-hosted-cp/values.schema.json @@ -100,8 +100,11 @@ "required": [ "vnetName", "nodeSubnetName", - "routeTableName", - "securityGroupName" + "nodeRouteTableName", + "nodeSecurityGroupName", + "cpSubnetName", + "cpRouteTableName", + "cpSecurityGroupName" ], "properties": { "vnetName": { diff --git a/templates/cluster/azure-hosted-cp/values.yaml b/templates/cluster/azure-hosted-cp/values.yaml index b4e1b81e..075fbb91 100644 --- a/templates/cluster/azure-hosted-cp/values.yaml +++ b/templates/cluster/azure-hosted-cp/values.yaml @@ -24,8 +24,11 @@ resourceGroup: "" network: vnetName: "" nodeSubnetName: "" - routeTableName: "" - securityGroupName: "" + nodeRouteTableName: "" + nodeSecurityGroupName: "" + cpSubnetName: "" + cpRouteTableName: "" + cpSecurityGroupName: "" # Azure machines parameters sshPublicKey: "" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e76a4c24..bc6ec680 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -30,6 +30,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/env" "k8s.io/utils/ptr" internalutils "github.com/Mirantis/hmc/internal/utils" @@ -48,12 +49,15 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace) - By("building and deploying the controller-manager") + By(fmt.Sprintf("building and deploying the controller-manager - Version: %s", + env.GetString("VERSION", ""))) cmd := exec.Command("make", "kind-deploy") - _, err := utils.Run(cmd) + output, err := utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) - cmd = exec.Command("make", "test-apply") - _, err = utils.Run(cmd) + cmd = exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") + output, err = utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) By("validating that the hmc-controller and CAPI provider controllers are running and ready") @@ -66,6 +70,15 @@ var _ = BeforeSuite(func() { } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + Eventually(func() error { + err = managedcluster.ValidateClusterTemplates(context.Background(), kc) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) }) var _ = AfterSuite(func() { @@ -93,6 +106,9 @@ func verifyControllersUp(kc *kubeclient.KubeClient) error { managedcluster.ProviderAWS, managedcluster.ProviderAzure, managedcluster.ProviderVSphere, + managedcluster.ProviderK0smotron, + managedcluster.ProviderK0smotronBootstrap, + managedcluster.ProviderK0smotronControlPlane, } for _, provider := range providers { diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index e3801e4e..b1dfc8fc 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -243,6 +243,21 @@ func (kc *KubeClient) listResource( return resources.Items, nil } +func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "clustertemplates", + }, true) + + resources, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list cluster templates") + } + + return resources.Items, nil +} + // ListMachines returns a list of Machine resources for the given cluster. func (kc *KubeClient) ListMachines(ctx context.Context, clusterName string) ([]unstructured.Unstructured, error) { GinkgoHelper() diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 441cfc49..3b051aa6 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -17,10 +17,14 @@ package aws import ( + "bufio" "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -52,14 +56,31 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - subnet, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue(), "failed to cast subnet to map") + type awsSubnetMaps []map[string]any + subnetMaps := make(awsSubnetMaps, len(subnets)) + for i, s := range subnets { + subnet, ok := s.(map[string]any) + Expect(ok).To(BeTrue(), "failed to cast subnet to map") + subnetMaps[i] = map[string]any{ + "isPublic": subnet["isPublic"], + "availabilityZone": subnet["availabilityZone"], + "id": subnet["resourceID"], + "routeTableId": subnet["routeTableId"], + "zoneType": "availability-zone", + } - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" { + subnetMaps[i]["natGatewayId"] = natGatewayID + } + } + var subnetsFormatted string + encodedYaml, err := yaml.Marshal(subnetMaps) + Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") + scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml))) + for scanner.Scan() { + subnetsFormatted += fmt.Sprintf(" %s\n", scanner.Text()) + } + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, subnetsFormatted) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") @@ -67,7 +88,5 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(managedcluster.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetAvailabilityZone, subnetAZ) GinkgoT().Setenv(managedcluster.EnvVarAWSSecurityGroupID, securityGroupID) } diff --git a/test/e2e/managedcluster/azure/azure.go b/test/e2e/managedcluster/azure/azure.go index 2880badc..9056e793 100644 --- a/test/e2e/managedcluster/azure/azure.go +++ b/test/e2e/managedcluster/azure/azure.go @@ -76,22 +76,37 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) resourceGroup := spec["resourceGroup"] GinkgoT().Setenv("AZURE_RESOURCE_GROUP", fmt.Sprintf("%s", resourceGroup)) - subnetMap, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue()) - subnetName := subnetMap["name"] - GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) - securityGroup, found, err := unstructured.NestedMap(subnetMap, "securityGroup") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - securityGroupName := securityGroup["name"] - GinkgoT().Setenv("AZURE_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + for _, subnet := range subnets { + sMap, ok := subnet.(map[string]any) + Expect(ok).To(BeTrue()) - routeTable, found, err := unstructured.NestedMap(subnetMap, "routeTable") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - routeTableName := routeTable["name"] - GinkgoT().Setenv("AZURE_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + routeTable, exists, err := unstructured.NestedMap(sMap, "routeTable") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + routeTableName := routeTable["name"] + + subnetName := sMap["name"] + + securityGroup, found, err := unstructured.NestedMap(sMap, "securityGroup") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + securityGroupName := securityGroup["name"] + + role, exists, err := unstructured.NestedString(sMap, "role") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + if role == "control-plane" { + GinkgoT().Setenv("AZURE_CP_SUBNET", fmt.Sprintf("%s", subnetName)) + GinkgoT().Setenv("AZURE_CP_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + GinkgoT().Setenv("AZURE_CP_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + } else { + GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) + GinkgoT().Setenv("AZURE_NODE_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + GinkgoT().Setenv("AZURE_NODE_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + } + } } // CreateDefaultStorageClass configures the default storage class for Azure diff --git a/test/e2e/managedcluster/clusteridentity/clusteridentity.go b/test/e2e/managedcluster/clusteridentity/clusteridentity.go index 41e01c77..bc05348d 100644 --- a/test/e2e/managedcluster/clusteridentity/clusteridentity.go +++ b/test/e2e/managedcluster/clusteridentity/clusteridentity.go @@ -134,7 +134,7 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust ci.createSecret(kc) ci.createClusterIdentity(kc) ci.createCredential(kc) - + ci.waitForCredentialReady(kc) return &ci } @@ -252,3 +252,40 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) { kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced) } + +func (ci *ClusterIdentity) waitForCredentialReady(kc *kubeclient.KubeClient) { + GinkgoHelper() + + By(fmt.Sprintf("waiting for %s credential to be ready", ci.Kind)) + + ctx := context.Background() + credName := fmt.Sprintf("%s-cred", ci.IdentityName) + + gvr := schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "credentials", + } + + Eventually(func() error { + creds, err := kc.GetDynamicClient(gvr, true).Get(ctx, credName, metav1.GetOptions{}) + if err != nil { + return err + } + + ready, found, err := unstructured.NestedBool(creds.Object, "status", "ready") + if err != nil { + return err + } + + if !found { + return fmt.Errorf("ready status not found on credential %s", credName) + } + + if !ready { + return fmt.Errorf("credential %s not ready", credName) + } + + return nil + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed()) +} diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go index 4f18a783..1d5d2104 100644 --- a/test/e2e/managedcluster/constants.go +++ b/test/e2e/managedcluster/constants.go @@ -25,15 +25,13 @@ const ( EnvVarNoCleanup = "NO_CLEANUP" // AWS - EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" - EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" - EnvVarAWSVPCID = "AWS_VPC_ID" - EnvVarAWSSubnetID = "AWS_SUBNET_ID" - EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE" - EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" - EnvVarAWSSecurityGroupID = "AWS_SG_ID" - EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarPublicIP = "AWS_PUBLIC_IP" + EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvVarAWSVPCID = "AWS_VPC_ID" + EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" + EnvVarAWSSecurityGroupID = "AWS_SG_ID" + EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" + EnvVarAWSSubnets = "AWS_SUBNETS" // VSphere EnvVarVSphereUser = "VSPHERE_USER" diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index 72efa96f..6555cf17 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -15,6 +15,7 @@ package managedcluster import ( + "context" _ "embed" "fmt" "os" @@ -26,17 +27,22 @@ import ( . "github.com/onsi/gomega" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/env" + "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/utils" ) type ProviderType string const ( - ProviderCAPI ProviderType = "cluster-api" - ProviderAWS ProviderType = "infrastructure-aws" - ProviderAzure ProviderType = "infrastructure-azure" - ProviderVSphere ProviderType = "infrastructure-vsphere" + ProviderCAPI ProviderType = "cluster-api" + ProviderAWS ProviderType = "infrastructure-aws" + ProviderAzure ProviderType = "infrastructure-azure" + ProviderVSphere ProviderType = "infrastructure-vsphere" + ProviderK0smotron ProviderType = "infrastructure-k0sproject-k0smotron" + ProviderK0smotronBootstrap ProviderType = "bootstrap-k0sproject-k0smotron" + ProviderK0smotronControlPlane ProviderType = "control-plane-k0sproject-k0smotron" providerLabel = "cluster.x-k8s.io/provider" ) @@ -121,9 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { // since we populate the vars from standalone prior to this step. ValidateDeploymentVars([]string{ EnvVarAWSVPCID, - EnvVarAWSSubnetID, - EnvVarAWSSubnetAvailabilityZone, - EnvVarAWSSecurityGroupID, + EnvVarAWSSubnets, }) managedClusterTemplateBytes = awsHostedCPManagedClusterTemplateBytes case TemplateVSphereStandaloneCP: @@ -138,17 +142,48 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } + version := env.GetString("VERSION", "") + index := strings.LastIndex(env.GetString("VERSION", ""), "-") + if index > 0 { + version = version[index:] + } + GinkgoT().Setenv("BUILD_VERSION", version) + managedClusterConfigBytes, err := envsubst.Bytes(managedClusterTemplateBytes) Expect(err).NotTo(HaveOccurred(), "failed to substitute environment variables") var managedClusterConfig map[string]any - + By(fmt.Sprintf("Cluster being applied\n %s", managedClusterConfigBytes)) err = yaml.Unmarshal(managedClusterConfigBytes, &managedClusterConfig) Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config") return &unstructured.Unstructured{Object: managedClusterConfig} } +func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error { + templates, err := client.ListClusterTemplates(ctx) + if err != nil { + return fmt.Errorf("failed to list cluster templates: %w", err) + } + + for _, template := range templates { + valid, found, err := unstructured.NestedBool(template.Object, "status", "valid") + if err != nil { + return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err) + } + + if !found { + return fmt.Errorf("valid flag for template %s not found", template.GetName()) + } + + if !valid { + return fmt.Errorf("template %s is still invalid", template.GetName()) + } + } + + return nil +} + func ValidateDeploymentVars(v []string) { GinkgoHelper() diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index 8a2700c6..cf67c0e1 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-hosted-cp-0-0-3 + template: aws-hosted-cp-0-0-3${BUILD_VERSION} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: @@ -12,8 +12,7 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - - id: ${AWS_SUBNET_ID} - availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} +${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} diff --git a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl index 24c449bc..3a42d828 100644 --- a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-standalone-cp-0-0-3 + template: aws-standalone-cp-0-0-3${BUILD_VERSION} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: @@ -15,5 +15,7 @@ spec: workersNumber: ${WORKERS_NUMBER:=1} controlPlane: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 32 worker: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 32 diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl index 76da17cb..919a7c25 100644 --- a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-hosted-cp-0-0-3 + template: azure-hosted-cp-0-0-3${BUILD_VERSION} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: location: "${AZURE_REGION}" @@ -17,8 +17,11 @@ spec: network: vnetName: "${AZURE_VM_NET_NAME}" nodeSubnetName: "${AZURE_NODE_SUBNET}" - routeTableName: "${AZURE_ROUTE_TABLE}" - securityGroupName: "${AZURE_SECURITY_GROUP}" + nodeRouteTableName: "${AZURE_NODE_ROUTE_TABLE}" + nodeSecurityGroupName: "${AZURE_NODE_SECURITY_GROUP}" + cpSubnetName: "${AZURE_CP_SUBNET}" + cpRouteTableName: "${AZURE_CP_ROUTE_TABLE}" + cpSecurityGroupName: "${AZURE_CP_SECURITY_GROUP}" tenantID: "${AZURE_TENANT_ID}" clientID: "${AZURE_CLIENT_ID}" clientSecret: "${AZURE_CLIENT_SECRET}" diff --git a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl index 3894b7b3..14e5bce6 100644 --- a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-standalone-cp-0-0-3 + template: azure-standalone-cp-0-0-3${BUILD_VERSION} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: 1 diff --git a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl index c0475f3f..3d2352e7 100644 --- a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-hosted-cp-0-0-3 + template: vsphere-hosted-cp-0-0-3${BUILD_VERSION} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl index cc5fa87b..90cbe942 100644 --- a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-standalone-cp-0-0-3 + template: vsphere-standalone-cp-0-0-3${BUILD_VERSION} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/managedcluster/validate_deployed.go b/test/e2e/managedcluster/validate_deployed.go index bae823f7..aa5fca51 100644 --- a/test/e2e/managedcluster/validate_deployed.go +++ b/test/e2e/managedcluster/validate_deployed.go @@ -266,6 +266,8 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str } for _, i := range service.Status.LoadBalancer.Ingress { + _, _ = fmt.Fprintf(GinkgoWriter, "HOSTNAME=%s, IP=%s", i.Hostname, i.IP) + _, _ = fmt.Fprintf(GinkgoWriter, "service status=%v", service.Status) if i.Hostname != "" || i.IP != "" { return nil } diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 6614698b..ee0abe61 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -124,6 +124,14 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index fcdbe27c..a3b5fd2c 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/utils/env" internalutils "github.com/Mirantis/hmc/internal/utils" "github.com/Mirantis/hmc/test/e2e/kubeclient" @@ -107,8 +108,9 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("Deploy onto standalone cluster") GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", "test-apply") - _, err := utils.Run(cmd) + cmd := exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") + output, err := utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) @@ -123,6 +125,16 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + By("Ensure cluster templates valid") + err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Create azure credential secret") clusteridentity.New(standaloneClient, managedcluster.ProviderAzure)