Merge pull request #636 from makhov/k0s-version-test-matrix

Test matrix for different k0s versions
k0sproject · Jul 12, 2024 · 86e831e · 86e831e
2 parents df6cbfa + b32e0ad
commit 86e831e
Show file tree

Hide file tree

Showing 7 changed files with 155 additions and 110 deletions.
diff --git a/.github/workflows/capi-smoke-tests.yml b/.github/workflows/capi-smoke-tests.yml
@@ -0,0 +1,107 @@
+name: Smoke tests
+
+on:
+  workflow_call:
+    inputs:
+      smoke-suite:
+        required: true
+        type: string
+      k0s-version:
+        required: false
+        type: string
+
+jobs:
+  capi-smokes:
+    name: "${{ inputs.smoke-suite }} :: ${{ inputs.k0s-version }}"
+    runs-on: ubuntu-22.04-8core
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Create image bundle
+        run: |
+          make release
+          make k0smotron-image-bundle.tar
+
+      - name: Create kind network with IPv4 only
+        run: |
+          docker network create kind --opt com.docker.network.bridge.enable_ip_masquerade=true
+
+
+      - name: Download kind
+        uses: supplypike/setup-bin@v4
+        with:
+          name: kind
+          version: v0.20.0
+          uri: https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64
+
+      - name: Setup KinD cluster
+        run: |
+          kind create cluster --config config/samples/capi/docker/kind.yaml
+
+      - name: Load k0smotron image to KinD
+        run: |
+          kind load image-archive k0smotron-image-bundle.tar
+
+      - name: Download clusterctl
+        uses: supplypike/setup-bin@v4
+        with:
+          name: clusterctl
+          version: v1.4.3
+          uri: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.3/clusterctl-linux-amd64
+
+      - name: Prepare cluster api components
+        run: |
+          make bootstrap-components.yaml control-plane-components.yaml infrastructure-components.yaml
+          mkdir -p v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/
+
+          mv bootstrap-components.yaml v0.0.0
+          mv control-plane-components.yaml v0.0.0
+          mv infrastructure-components.yaml v0.0.0
+          mv ./hack/capi-ci/metadata.yaml v0.0.0
+
+          cp -r v0.0.0 k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/
+          cp -r v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/
+          cp -r v0.0.0 k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/
+          sed -e 's#%pwd%#'`pwd`'#g' ./hack/capi-ci/config.yaml > config.yaml
+
+      - name: Install cluster api components
+        run: |
+          clusterctl init --control-plane k0sproject-k0smotron --bootstrap k0sproject-k0smotron --infrastructure k0sproject-k0smotron,docker --config config.yaml
+          kubectl patch -n capi-system deployment/capi-controller-manager -p \
+            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
+          kubectl patch -n capd-system deployment/capd-controller-manager -p \
+            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
+
+      - name: Install PVC provider
+        run: |
+          kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.24/deploy/local-path-storage.yaml
+
+      - name: Run inttest for CAPI with docker provider
+        run: |
+          kind get kubeconfig > kind.conf
+          export KUBECONFIG=$(realpath kind.conf)
+          docker system prune -f
+          make -C inttest ${{ inputs.smoke-suite }} K0S_VERSION=${{ inputs.k0s-version }}
+
+      - name: Get k0smotron logs
+        if: failure()
+        run: |
+          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-bootstrap > /tmp/${{ inputs.smoke-suite }}-k0smotron-bootstrap.log
+          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-control-plane > /tmp/${{ inputs.smoke-suite }}-k0smotron-control-plane.log
+          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-infrastructure > /tmp/${{ inputs.smoke-suite }}-k0smotron-infrastructure.log
+
+      - name: Collect k0s logs and support bundle
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoketests-${{ inputs.smoke-suite }}-files
+          path: |
+            /tmp/${{ inputs.smoke-suite }}-k0smotron-*.log
+    
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -123,10 +123,26 @@ jobs:
         run: |
           make -C inttest ${{ matrix.smoke-suite }}
 
+
+  capi-smokes-on-different-k0s-versions:
+    name: CAPI Smokes Matrix
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        smoke-suite:
+          - check-capi-remote-machine
+          - check-capi-remote-machine-template
+        k0s-version: [ "v1.28.11", "v1.29.6", "v1.30.2" ]
+
+    uses: ./.github/workflows/capi-smoke-tests.yml
+    with:
+      smoke-suite: ${{ matrix.smoke-suite }}
+      k0s-version: ${{ matrix.k0s-version }}
+
   capi-smokes:
     name: Cluster API smoke tests
     needs: build
-    runs-on: ubuntu-22.04-8core
     strategy:
       fail-fast: false
       matrix:
@@ -141,97 +157,10 @@ jobs:
           - check-capi-controlplane-docker-tunneling
           - check-capi-controlplane-docker-tunneling-proxy
           - check-capi-controlplane-docker-worker
-          - check-capi-remote-machine
-          - check-capi-remote-machine-template
           - check-capi-remote-machine-template-update
           - check-capi-docker-machine-template-update
           - check-capi-remote-machine-job-provision
-    steps:
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-
-      - name: Create image bundle
-        run: |
-          make release
-          make k0smotron-image-bundle.tar
-
-      - name: Create kind network with IPv4 only
-        run: |
-          docker network create kind --opt com.docker.network.bridge.enable_ip_masquerade=true
-
-
-      - name: Download kind
-        uses: supplypike/setup-bin@v4
-        with:
-          name: kind
-          version: v0.20.0
-          uri: https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64
-
-      - name: Setup KinD cluster
-        run: |
-          kind create cluster --config config/samples/capi/docker/kind.yaml
-
-      - name: Load k0smotron image to KinD
-        run: |
-          kind load image-archive k0smotron-image-bundle.tar
-
-      - name: Download clusterctl
-        uses: supplypike/setup-bin@v4
-        with:
-          name: clusterctl
-          version: v1.4.3
-          uri: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.3/clusterctl-linux-amd64
-
-      - name: Prepare cluster api components
-        run: |
-          make bootstrap-components.yaml control-plane-components.yaml infrastructure-components.yaml
-          mkdir -p v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/
-          
-          mv bootstrap-components.yaml v0.0.0
-          mv control-plane-components.yaml v0.0.0
-          mv infrastructure-components.yaml v0.0.0
-          mv ./hack/capi-ci/metadata.yaml v0.0.0
-          
-          cp -r v0.0.0 k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/
-          cp -r v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/
-          cp -r v0.0.0 k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/
-          sed -e 's#%pwd%#'`pwd`'#g' ./hack/capi-ci/config.yaml > config.yaml
-
-      - name: Install cluster api components
-        run: |
-          clusterctl init --control-plane k0sproject-k0smotron --bootstrap k0sproject-k0smotron --infrastructure k0sproject-k0smotron,docker --config config.yaml
-          kubectl patch -n capi-system deployment/capi-controller-manager -p \
-            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
-          kubectl patch -n capd-system deployment/capd-controller-manager -p \
-            '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}'
-
-      - name: Install PVC provider
-        run: |
-          kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.24/deploy/local-path-storage.yaml
 
-      - name: Run inttest for CAPI with docker provider
-        run: |
-          kind get kubeconfig > kind.conf
-          export KUBECONFIG=$(realpath kind.conf)
-          docker system prune -f
-          make -C inttest ${{ matrix.smoke-suite }}
-
-      - name: Get k0smotron logs
-        if: failure()
-        run: |
-          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-bootstrap > /tmp/${{ matrix.smoke-suite }}-k0smotron-bootstrap.log
-          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-control-plane > /tmp/${{ matrix.smoke-suite }}-k0smotron-control-plane.log
-          kubectl logs -n k0smotron deploy/k0smotron-controller-manager-infrastructure > /tmp/${{ matrix.smoke-suite }}-k0smotron-infrastructure.log
-
-      - name: Collect k0s logs and support bundle
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: smoketests-${{ matrix.smoke-suite }}-files
-          path: |
-            /tmp/${{ matrix.smoke-suite }}-k0smotron-*.log
+    uses: ./.github/workflows/capi-smoke-tests.yml
+    with:
+      smoke-suite: ${{ matrix.smoke-suite }}
diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go
@@ -306,7 +306,7 @@ func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Con
 			Name:            "dns-check",
 			Image:           kmc.Spec.GetImage(),
 			ImagePullPolicy: v1.PullIfNotPresent,
-			Command:         []string{"/bin/bash", "-c"},
+			Command:         []string{"/bin/sh", "-c"},
 			Args:            []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"},
 			Env: []v1.EnvVar{
 				{Name: "SVC_NAME", Value: kmc.GetEtcdServiceName()},

diff --git a/internal/util/dynamic_config.go b/internal/util/dynamic_config.go
@@ -25,7 +25,6 @@ func ReconcileDynamicConfig(ctx context.Context, cluster metav1.Object, cli clie
 	if err != nil {
 		return fmt.Errorf("failed to marshal unstructured config: %w", err)
 	}
-	fmt.Println(string(b))
 
 	chCS, err := remote.NewClusterClient(ctx, "k0smotron", cli, util.ObjectKey(cluster))
 	if err != nil {

diff --git a/inttest/Makefile b/inttest/Makefile
@@ -5,6 +5,7 @@ K0SMOTRON_IMAGES_BUNDLE ?= $(realpath ../k0smotron-image-bundle.tar)
 K0SMOTRON_INSTALL_YAML ?= $(realpath ../install.yaml)
 LOCAL_STORAGE_INSTALL_YAML ?= $(realpath ./footloose-alpine/local-path-storage.yaml)
 K0S_USE_DEFAULT_K0S_BINARIES ?= true
+K0S_VERSION ?= v1.30.2
 TIMEOUT ?= 6m
 
 .PHONY: $(smoketests)
@@ -28,6 +29,7 @@ $(smoketests): ../k0smotron-image-bundle.tar ../install.yaml .footloose-alpine.s
 	FOOTLOOSE_IMAGE='$(FOOTLOOSE_IMAGE)' \
 	K0SMOTRON_IMAGES_BUNDLE='$(K0SMOTRON_IMAGES_BUNDLE)' \
 	K0SMOTRON_INSTALL_YAML='$(K0SMOTRON_INSTALL_YAML)' \
+	K0S_VERSION='$(K0S_VERSION)' \
 	LOCAL_STORAGE_INSTALL_YAML='$(LOCAL_STORAGE_INSTALL_YAML)' \
 	K0S_USE_DEFAULT_K0S_BINARIES=$(K0S_USE_DEFAULT_K0S_BINARIES) \
 	go test -count=1 -v -timeout $(TIMEOUT) github.com/k0sproject/k0smotron/inttest/$(TEST_PACKAGE)

diff --git a/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go b/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go
@@ -226,17 +226,21 @@ func (s *RemoteMachineTemplateSuite) createCluster() {
 	s.Require().NoError(err)
 
 	// Execute the template to buffer
-	var clusterYaml bytes.Buffer
+	var clusterYamlBuf bytes.Buffer
 
-	err = t.Execute(&clusterYaml, struct {
-		Address string
-		SSHKey  string
+	k0sVersion := os.Getenv("K0S_VERSION")
+
+	err = t.Execute(&clusterYamlBuf, struct {
+		Address    string
+		SSHKey     string
+		K0SVersion string
 	}{
-		Address: workerIP,
-		SSHKey:  base64.StdEncoding.EncodeToString(s.privateKey),
+		Address:    workerIP,
+		SSHKey:     base64.StdEncoding.EncodeToString(s.privateKey),
+		K0SVersion: k0sVersion,
 	})
 	s.Require().NoError(err)
-	bytes := clusterYaml.Bytes()
+	bytes := clusterYamlBuf.Bytes()
 
 	s.Require().NoError(os.WriteFile(s.clusterYamlsPath, bytes, 0644))
 	out, err := exec.Command("kubectl", "apply", "-f", s.clusterYamlsPath).CombinedOutput()
@@ -276,7 +280,7 @@ metadata:
   name: remote-test
 spec:
   replicas: 1
-  version: v1.27.1+k0s.0
+  version: {{ .K0SVersion }}+k0s.0
   k0sConfigSpec:
     k0s:
       apiVersion: k0s.k0sproject.io/v1beta1

diff --git a/inttest/capi-remote-machine/capi_remote_machine_test.go b/inttest/capi-remote-machine/capi_remote_machine_test.go
@@ -222,18 +222,22 @@ func (s *RemoteMachineSuite) createCluster() {
 	t, err := template.New("cluster").Parse(clusterYaml)
 	s.Require().NoError(err)
 
+	k0sVersion := os.Getenv("K0S_VERSION")
+
 	// Execute the template to buffer
-	var clusterYaml bytes.Buffer
+	var clusterYamlBuf bytes.Buffer
 
-	err = t.Execute(&clusterYaml, struct {
-		Address string
-		SSHKey  string
+	err = t.Execute(&clusterYamlBuf, struct {
+		Address    string
+		SSHKey     string
+		K0SVersion string
 	}{
-		Address: workerIP,
-		SSHKey:  base64.StdEncoding.EncodeToString(s.privateKey),
+		Address:    workerIP,
+		SSHKey:     base64.StdEncoding.EncodeToString(s.privateKey),
+		K0SVersion: k0sVersion,
 	})
 	s.Require().NoError(err)
-	bytes := clusterYaml.Bytes()
+	bytes := clusterYamlBuf.Bytes()
 	// s.T().Logf("cluster yaml: %s", string(bytes))
 	resources, err := util.ParseManifests(bytes)
 	s.Require().NoError(err)
@@ -287,7 +291,7 @@ metadata:
   name: remote-test
   namespace: default
 spec:
-  version: v1.27.2-k0s.0
+  version: {{ .K0SVersion }}-k0s.0
   persistence:
     type: emptyDir
   service:
@@ -323,7 +327,7 @@ metadata:
   name: remote-test-0
   namespace: default
 spec:
-  version: v1.27.2+k0s.0
+  version: {{ .K0SVersion }}+k0s.0
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: RemoteMachine