Add e2e using capi framework

Signed-off-by: Adrian Pedriza <[email protected]>

.github/workflows/capi-smoke-tests.yml

@@ -80,11 +80,7 @@ jobs:
       - name: Install cluster api components
         run: |
           clusterctl init --control-plane k0sproject-k0smotron --bootstrap k0sproject-k0smotron --infrastructure k0sproject-k0smotron,docker --config config.yaml
-          kubectl patch -n capi-system deployment/capi-controller-manager -p \
-            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
-          kubectl patch -n capd-system deployment/capd-controller-manager -p \
-            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
-
+        
       - name: Install PVC provider
         run: |
           kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.24/deploy/local-path-storage.yaml

.github/workflows/go.yml

@@ -169,3 +169,31 @@ jobs:
     uses: ./.github/workflows/capi-smoke-tests.yml
     with:
       smoke-suite: ${{ matrix.smoke-suite }}
+  e2e-migration:
+    name: E2E migration (WIP)
+    needs: build
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Build e2e images
+      run: make docker-build
+
+    - name: Run e2e tests 
+      run: make test-e2e
+
+    - name: Archive artifacts
+      if: failure()
+      uses: actions/[email protected]
+      with:
+        name: e2e-artifacts
+        path: _artifacts
+        if-no-files-found: ignore

.gitignore

@@ -32,3 +32,6 @@ k0smotron-image-bundle.tar
 *.swp
 *.swo
 *~
+
+# test results
+_artifacts

Makefile

@@ -13,6 +13,18 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 CRDOC ?= $(LOCALBIN)/crdoc
+GINKGO ?= $(LOCALBIN)/ginkgo
+
+#
+# Ginkgo configuration.
+#
+GINKGO_POLL_PROGRESS_AFTER ?= 60m
+GINKGO_POLL_PROGRESS_INTERVAL ?= 5m
+GINKGO_TIMEOUT ?= 2h
+E2E_CONF_FILE ?= $(shell pwd)/e2e/config/docker.yaml
+SKIP_RESOURCE_CLEANUP ?= false
+# Artifacts folder generated for e2e tests
+ARTIFACTS ?= $(shell pwd)/_artifacts
 
 # Image URL to use all building/pushing image targets
 IMG ?= quay.io/k0sproject/k0smotron:latest
@@ -100,6 +112,17 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(GO_TEST_DIRS) -coverprofile cover.out
 
+.PHONY: test-e2e
+test-e2e: $(GINKGO) ## Run the end-to-end tests
+	$(GINKGO) -v --trace \
+		-poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
+		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) \
+		--timeout=$(GINKGO_TIMEOUT) \
+		--output-dir="$(ARTIFACTS)" $(shell pwd)/e2e -- \
+	    -e2e.artifacts-folder="$(ARTIFACTS)" \
+	    -e2e.config="$(E2E_CONF_FILE)" \
+	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP)
+
 ##@ Build
 
 .PHONY: build
@@ -208,6 +231,11 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
+.PHONY: ginkgo
+ginkgo: $(GINKGO) ## Download ginkgo locally if necessary.
+$(GINKGO): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo
+
 .PHONY: docs
 docs:
 	$(MAKE) -C docs

e2e/common.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+// Test suite constants for e2e config variables.
+const (
+	KubernetesVersion                = "KUBERNETES_VERSION"
+	KubernetesVersionManagement      = "KUBERNETES_VERSION_MANAGEMENT"
+	KubernetesVersionFirstUpgradeTo  = "KUBERNETES_VERSION_FIRST_UPGRADE_TO"
+	KubernetesVersionSecondUpgradeTo = "KUBERNETES_VERSION_SECOND_UPGRADE_TO"
+	ControlPlaneMachineCount         = "CONTROL_PLANE_MACHINE_COUNT"
+	IPFamily                         = "IP_FAMILY"
+)

e2e/config/docker.yaml

@@ -0,0 +1,88 @@
+---
+# E2E test scenario using local dev images and manifests built from the source tree for following providers:
+# - cluster-api
+# - bootstrap k0smotron
+# - control-plane k0smotron
+# - infrastructure docker
+images:
+  - name: quay.io/k0sproject/k0smotron
+    loadBehavior: mustLoad
+
+providers:
+  - name: cluster-api
+    type: CoreProvider
+    versions:
+      - name: v1.8.5
+        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.5/core-components.yaml
+        type: url
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+  - name: docker
+    type: InfrastructureProvider
+    versions:
+      - name: v1.8.1
+        value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.8.1/infrastructure-components-development.yaml
+        type: url
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/v1beta1/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+    files:
+      - sourcePath: "../data/infrastructure-docker/cluster-template-ooc.yaml"
+  - name: k0sproject-k0smotron
+    type: ControlPlaneProvider
+    versions:
+      - name: v1.2.99
+        value: ../../config/default
+        contract: v1beta1
+        files:
+          - sourcePath: "../../metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+          - old: "image: k0s/k0smotron:latest"
+            new: "image: quay.io/k0sproject/k0smotron:latest"
+
+variables:
+  KUBERNETES_VERSION_MANAGEMENT: "v1.30.0"
+  KUBERNETES_VERSION: "v1.31.0"
+  KUBERNETES_VERSION_FIRST_UPGRADE_TO: "v1.30.2+k0s.0"
+  KUBERNETES_VERSION_SECOND_UPGRADE_TO: "v1.31.2+k0s.0"
+  IP_FAMILY: "IPv4"
+  KIND_IMAGE_VERSION: "v1.30.0"
+  # Used during clusterctl upgrade test
+  CAPI_CORE_VERSION: "1.8.5"
+  # Enabling the feature flags by setting the env variables.
+  CLUSTER_TOPOLOGY: "true"
+  EXP_MACHINE_POOL: "true"
+
+intervals:
+  # The array is defined as [timeout, polling interval]
+  # copied from https://github.com/kubernetes-sigs/cluster-api/blob/main/test/e2e/config/docker.yaml
+  default/wait-controllers: ["3m", "10s"]
+  default/wait-cluster: ["5m", "10s"]
+  default/wait-control-plane: ["10m", "10s"]
+  default/wait-worker-nodes: ["10m", "10s"]
+  default/wait-machine-pool-nodes: ["10m", "10s"]
+  default/wait-delete-cluster: ["3m", "10s"]
+  default/wait-kube-proxy-upgrade: ["30m", "10s"]
+  default/wait-machine-pool-upgrade: ["30m", "10s"]
+  default/wait-nodes-ready: ["10m", "10s"]
+  default/wait-machine-remediation: ["5m", "10s"]
+  default/wait-autoscaler: ["5m", "10s"]
+  node-drain/wait-deployment-available: ["3m", "10s"]
+  node-drain/wait-control-plane: ["15m", "10s"]
+  node-drain/wait-machine-deleted: ["2m", "10s"]
+  kcp-remediation/wait-machines: ["5m", "10s"]
+  kcp-remediation/check-machines-stable: ["30s", "5s"]
+  kcp-remediation/wait-machine-provisioned: ["5m", "10s"]
+  #  Giving a bit more time during upgrade tests
+  workload-upgrade/wait-cluster: ["10m", "10s"]
+  workload-upgrade/wait-control-plane: ["20m", "10s"]
+  workload-upgrade/wait-worker-nodes: ["20m", "10s"]

e2e/data/infrastructure-docker/cluster-template-ooc.yaml

@@ -0,0 +1,135 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 192.168.0.0/16
+    serviceDomain: cluster.local
+    services:
+      cidrBlocks:
+      - 10.128.0.0/12
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: K0sControlPlane
+    name: ${CLUSTER_NAME}-docker-test
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: docker-test
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: docker-test-cp-template
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      customImage: kindest/node:v1.31.0
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: K0sControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-docker-test
+  namespace: ${NAMESPACE}
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: v1.30.1+k0s.0
+  k0sConfigSpec:
+    args:
+    - --enable-worker
+    k0s:
+      apiVersion: k0s.k0sproject.io/v1beta1
+      kind: ClusterConfig
+      metadata:
+        name: k0s
+      spec:
+        api:
+          extraArgs:
+            anonymous-auth: "true"
+        telemetry:
+          enabled: false
+        network:
+          controlPlaneLoadBalancing:
+            enabled: false 
+    files:
+    - path: /tmp/test-file-secret
+      contentFrom: 
+        secretRef: 
+          name: test-file-secret
+          key: value
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: docker-test-cp-template
+      namespace: ${NAMESPACE}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: docker-test
+  namespace: ${NAMESPACE}
+spec:
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Machine
+metadata:
+  name:  ${CLUSTER_NAME}-docker-test-worker-0
+  namespace: ${NAMESPACE}
+spec:
+  version: v1.30.1
+  clusterName: ${CLUSTER_NAME}
+  bootstrap:
+    configRef:
+      apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+      kind: K0sWorkerConfig
+      name: docker-test-worker-0
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachine
+    name: ${CLUSTER_NAME}-docker-test-worker-0
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: K0sWorkerConfig
+metadata:
+  name: docker-test-worker-0
+  namespace: ${NAMESPACE}
+spec:
+  # version is deliberately different to be able to verify we actually pick it up :)
+  version: v1.30.1+k0s.0
+  args:
+    - --labels=k0sproject.io/foo=bar
+  preStartCommands:
+    - echo -n "pre-start" > /tmp/pre-start
+  postStartCommands:
+    - echo -n "post-start" > /tmp/post-start
+  files:
+    - path: /tmp/test-file
+      content: test-file
+    - path: /tmp/test-file-secret
+      contentFrom: 
+        secretRef: 
+          name: test-file-secret
+          key: value
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachine
+metadata:
+  name: ${CLUSTER_NAME}-docker-test-worker-0
+  namespace: ${NAMESPACE}
+spec:
+  customImage: kindest/node:v1.31.0
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: test-file-secret
+  namespace: ${NAMESPACE}
+type: Opaque
+data:
+  value: dGVzdA==

e2e/data/shared/v1beta1/metadata.yaml

@@ -0,0 +1,9 @@
+# maps release series of major.minor to cluster-api contract version,
+# update this file only when you update the version for cluster-api
+# CoreProvider and docker InfrastructureProvider in test/e2e/config/k3s-docker.yaml
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+releaseSeries:
+  - major: 1
+    minor: 8
+    contract: v1beta1