From b32e0adf8432d77d74932d66cfdd114b534aebd5 Mon Sep 17 00:00:00 2001 From: Alexey Makhov Date: Fri, 28 Jun 2024 13:10:43 +0300 Subject: [PATCH] Test matrix for different k0s versions Signed-off-by: Alexey Makhov --- .github/workflows/capi-smoke-tests.yml | 107 +++++++++++++++++ .github/workflows/go.yml | 111 ++++-------------- .../k0smotron.io/k0smotroncluster_etcd.go | 2 +- internal/util/dynamic_config.go | 1 - inttest/Makefile | 2 + .../capi_remote_machine_template_test.go | 20 ++-- .../capi_remote_machine_test.go | 22 ++-- 7 files changed, 155 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/capi-smoke-tests.yml diff --git a/.github/workflows/capi-smoke-tests.yml b/.github/workflows/capi-smoke-tests.yml new file mode 100644 index 000000000..2e395799f --- /dev/null +++ b/.github/workflows/capi-smoke-tests.yml @@ -0,0 +1,107 @@ +name: Smoke tests + +on: + workflow_call: + inputs: + smoke-suite: + required: true + type: string + k0s-version: + required: false + type: string + +jobs: + capi-smokes: + name: "${{ inputs.smoke-suite }} :: ${{ inputs.k0s-version }}" + runs-on: ubuntu-22.04-8core + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Create image bundle + run: | + make release + make k0smotron-image-bundle.tar + + - name: Create kind network with IPv4 only + run: | + docker network create kind --opt com.docker.network.bridge.enable_ip_masquerade=true + + + - name: Download kind + uses: supplypike/setup-bin@v4 + with: + name: kind + version: v0.20.0 + uri: https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64 + + - name: Setup KinD cluster + run: | + kind create cluster --config config/samples/capi/docker/kind.yaml + + - name: Load k0smotron image to KinD + run: | + kind load image-archive k0smotron-image-bundle.tar + + - name: Download clusterctl + uses: supplypike/setup-bin@v4 + with: + name: clusterctl + version: v1.4.3 + uri: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.3/clusterctl-linux-amd64 + + - name: Prepare cluster api components + run: | + make bootstrap-components.yaml control-plane-components.yaml infrastructure-components.yaml + mkdir -p v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/ + + mv bootstrap-components.yaml v0.0.0 + mv control-plane-components.yaml v0.0.0 + mv infrastructure-components.yaml v0.0.0 + mv ./hack/capi-ci/metadata.yaml v0.0.0 + + cp -r v0.0.0 k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ + cp -r v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ + cp -r v0.0.0 k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/ + sed -e 's#%pwd%#'`pwd`'#g' ./hack/capi-ci/config.yaml > config.yaml + + - name: Install cluster api components + run: | + clusterctl init --control-plane k0sproject-k0smotron --bootstrap k0sproject-k0smotron --infrastructure k0sproject-k0smotron,docker --config config.yaml + kubectl patch -n capi-system deployment/capi-controller-manager -p \ + '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}' + kubectl patch -n capd-system deployment/capd-controller-manager -p \ + '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}' + + - name: Install PVC provider + run: | + kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.24/deploy/local-path-storage.yaml + + - name: Run inttest for CAPI with docker provider + run: | + kind get kubeconfig > kind.conf + export KUBECONFIG=$(realpath kind.conf) + docker system prune -f + make -C inttest ${{ inputs.smoke-suite }} K0S_VERSION=${{ inputs.k0s-version }} + + - name: Get k0smotron logs + if: failure() + run: | + kubectl logs -n k0smotron deploy/k0smotron-controller-manager-bootstrap > /tmp/${{ inputs.smoke-suite }}-k0smotron-bootstrap.log + kubectl logs -n k0smotron deploy/k0smotron-controller-manager-control-plane > /tmp/${{ inputs.smoke-suite }}-k0smotron-control-plane.log + kubectl logs -n k0smotron deploy/k0smotron-controller-manager-infrastructure > /tmp/${{ inputs.smoke-suite }}-k0smotron-infrastructure.log + + - name: Collect k0s logs and support bundle + if: failure() + uses: actions/upload-artifact@v4 + with: + name: smoketests-${{ inputs.smoke-suite }}-files + path: | + /tmp/${{ inputs.smoke-suite }}-k0smotron-*.log + diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 70b6f0c64..d3daa7988 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -123,10 +123,26 @@ jobs: run: | make -C inttest ${{ matrix.smoke-suite }} + + capi-smokes-on-different-k0s-versions: + name: CAPI Smokes Matrix + needs: build + strategy: + fail-fast: false + matrix: + smoke-suite: + - check-capi-remote-machine + - check-capi-remote-machine-template + k0s-version: [ "v1.28.11", "v1.29.6", "v1.30.2" ] + + uses: ./.github/workflows/capi-smoke-tests.yml + with: + smoke-suite: ${{ matrix.smoke-suite }} + k0s-version: ${{ matrix.k0s-version }} + capi-smokes: name: Cluster API smoke tests needs: build - runs-on: ubuntu-22.04-8core strategy: fail-fast: false matrix: @@ -141,97 +157,10 @@ jobs: - check-capi-controlplane-docker-tunneling - check-capi-controlplane-docker-tunneling-proxy - check-capi-controlplane-docker-worker - - check-capi-remote-machine - - check-capi-remote-machine-template - check-capi-remote-machine-template-update - check-capi-docker-machine-template-update - check-capi-remote-machine-job-provision - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Create image bundle - run: | - make release - make k0smotron-image-bundle.tar - - - name: Create kind network with IPv4 only - run: | - docker network create kind --opt com.docker.network.bridge.enable_ip_masquerade=true - - - - name: Download kind - uses: supplypike/setup-bin@v4 - with: - name: kind - version: v0.20.0 - uri: https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64 - - - name: Setup KinD cluster - run: | - kind create cluster --config config/samples/capi/docker/kind.yaml - - - name: Load k0smotron image to KinD - run: | - kind load image-archive k0smotron-image-bundle.tar - - - name: Download clusterctl - uses: supplypike/setup-bin@v4 - with: - name: clusterctl - version: v1.4.3 - uri: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.3/clusterctl-linux-amd64 - - - name: Prepare cluster api components - run: | - make bootstrap-components.yaml control-plane-components.yaml infrastructure-components.yaml - mkdir -p v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/ - - mv bootstrap-components.yaml v0.0.0 - mv control-plane-components.yaml v0.0.0 - mv infrastructure-components.yaml v0.0.0 - mv ./hack/capi-ci/metadata.yaml v0.0.0 - - cp -r v0.0.0 k0sproject-k0smotron/bootstrap-k0sproject-k0smotron/ - cp -r v0.0.0 k0sproject-k0smotron/control-plane-k0sproject-k0smotron/ - cp -r v0.0.0 k0sproject-k0smotron/infrastructure-k0sproject-k0smotron/ - sed -e 's#%pwd%#'`pwd`'#g' ./hack/capi-ci/config.yaml > config.yaml - - - name: Install cluster api components - run: | - clusterctl init --control-plane k0sproject-k0smotron --bootstrap k0sproject-k0smotron --infrastructure k0sproject-k0smotron,docker --config config.yaml - kubectl patch -n capi-system deployment/capi-controller-manager -p \ - '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}' - kubectl patch -n capd-system deployment/capd-controller-manager -p \ - '{"spec":{"template":{"spec":{"containers":[{"name":"manager","args":["--leader-elect", "--metrics-bind-addr=localhost:8080", "--feature-gates=ClusterTopology=true"]}]}}}}' - - - name: Install PVC provider - run: | - kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.24/deploy/local-path-storage.yaml - - name: Run inttest for CAPI with docker provider - run: | - kind get kubeconfig > kind.conf - export KUBECONFIG=$(realpath kind.conf) - docker system prune -f - make -C inttest ${{ matrix.smoke-suite }} - - - name: Get k0smotron logs - if: failure() - run: | - kubectl logs -n k0smotron deploy/k0smotron-controller-manager-bootstrap > /tmp/${{ matrix.smoke-suite }}-k0smotron-bootstrap.log - kubectl logs -n k0smotron deploy/k0smotron-controller-manager-control-plane > /tmp/${{ matrix.smoke-suite }}-k0smotron-control-plane.log - kubectl logs -n k0smotron deploy/k0smotron-controller-manager-infrastructure > /tmp/${{ matrix.smoke-suite }}-k0smotron-infrastructure.log - - - name: Collect k0s logs and support bundle - if: failure() - uses: actions/upload-artifact@v4 - with: - name: smoketests-${{ matrix.smoke-suite }}-files - path: | - /tmp/${{ matrix.smoke-suite }}-k0smotron-*.log + uses: ./.github/workflows/capi-smoke-tests.yml + with: + smoke-suite: ${{ matrix.smoke-suite }} diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go index 397aee19a..0badaf43b 100644 --- a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go +++ b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go @@ -306,7 +306,7 @@ func (r *ClusterReconciler) generateEtcdInitContainers(kmc *km.Cluster) []v1.Con Name: "dns-check", Image: kmc.Spec.GetImage(), ImagePullPolicy: v1.PullIfNotPresent, - Command: []string{"/bin/bash", "-c"}, + Command: []string{"/bin/sh", "-c"}, Args: []string{"getent ahostsv4 ${HOSTNAME}.${SVC_NAME}." + kmc.Namespace + ".svc"}, Env: []v1.EnvVar{ {Name: "SVC_NAME", Value: kmc.GetEtcdServiceName()}, diff --git a/internal/util/dynamic_config.go b/internal/util/dynamic_config.go index 32a809f97..7a00ad0b5 100644 --- a/internal/util/dynamic_config.go +++ b/internal/util/dynamic_config.go @@ -25,7 +25,6 @@ func ReconcileDynamicConfig(ctx context.Context, cluster metav1.Object, cli clie if err != nil { return fmt.Errorf("failed to marshal unstructured config: %w", err) } - fmt.Println(string(b)) chCS, err := remote.NewClusterClient(ctx, "k0smotron", cli, util.ObjectKey(cluster)) if err != nil { diff --git a/inttest/Makefile b/inttest/Makefile index d81c77bc5..d3965e625 100644 --- a/inttest/Makefile +++ b/inttest/Makefile @@ -5,6 +5,7 @@ K0SMOTRON_IMAGES_BUNDLE ?= $(realpath ../k0smotron-image-bundle.tar) K0SMOTRON_INSTALL_YAML ?= $(realpath ../install.yaml) LOCAL_STORAGE_INSTALL_YAML ?= $(realpath ./footloose-alpine/local-path-storage.yaml) K0S_USE_DEFAULT_K0S_BINARIES ?= true +K0S_VERSION ?= v1.30.2 TIMEOUT ?= 6m .PHONY: $(smoketests) @@ -28,6 +29,7 @@ $(smoketests): ../k0smotron-image-bundle.tar ../install.yaml .footloose-alpine.s FOOTLOOSE_IMAGE='$(FOOTLOOSE_IMAGE)' \ K0SMOTRON_IMAGES_BUNDLE='$(K0SMOTRON_IMAGES_BUNDLE)' \ K0SMOTRON_INSTALL_YAML='$(K0SMOTRON_INSTALL_YAML)' \ + K0S_VERSION='$(K0S_VERSION)' \ LOCAL_STORAGE_INSTALL_YAML='$(LOCAL_STORAGE_INSTALL_YAML)' \ K0S_USE_DEFAULT_K0S_BINARIES=$(K0S_USE_DEFAULT_K0S_BINARIES) \ go test -count=1 -v -timeout $(TIMEOUT) github.com/k0sproject/k0smotron/inttest/$(TEST_PACKAGE) diff --git a/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go b/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go index 030352bf6..239a7447e 100644 --- a/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go +++ b/inttest/capi-remote-machine-template/capi_remote_machine_template_test.go @@ -216,17 +216,21 @@ func (s *RemoteMachineTemplateSuite) createCluster() { s.Require().NoError(err) // Execute the template to buffer - var clusterYaml bytes.Buffer + var clusterYamlBuf bytes.Buffer - err = t.Execute(&clusterYaml, struct { - Address string - SSHKey string + k0sVersion := os.Getenv("K0S_VERSION") + + err = t.Execute(&clusterYamlBuf, struct { + Address string + SSHKey string + K0SVersion string }{ - Address: workerIP, - SSHKey: base64.StdEncoding.EncodeToString(s.privateKey), + Address: workerIP, + SSHKey: base64.StdEncoding.EncodeToString(s.privateKey), + K0SVersion: k0sVersion, }) s.Require().NoError(err) - bytes := clusterYaml.Bytes() + bytes := clusterYamlBuf.Bytes() s.Require().NoError(os.WriteFile(s.clusterYamlsPath, bytes, 0644)) out, err := exec.Command("kubectl", "apply", "-f", s.clusterYamlsPath).CombinedOutput() @@ -266,7 +270,7 @@ metadata: name: remote-test spec: replicas: 1 - version: v1.27.1+k0s.0 + version: {{ .K0SVersion }}+k0s.0 k0sConfigSpec: k0s: apiVersion: k0s.k0sproject.io/v1beta1 diff --git a/inttest/capi-remote-machine/capi_remote_machine_test.go b/inttest/capi-remote-machine/capi_remote_machine_test.go index 78119b699..2d345f0eb 100644 --- a/inttest/capi-remote-machine/capi_remote_machine_test.go +++ b/inttest/capi-remote-machine/capi_remote_machine_test.go @@ -209,18 +209,22 @@ func (s *RemoteMachineSuite) createCluster() { t, err := template.New("cluster").Parse(clusterYaml) s.Require().NoError(err) + k0sVersion := os.Getenv("K0S_VERSION") + // Execute the template to buffer - var clusterYaml bytes.Buffer + var clusterYamlBuf bytes.Buffer - err = t.Execute(&clusterYaml, struct { - Address string - SSHKey string + err = t.Execute(&clusterYamlBuf, struct { + Address string + SSHKey string + K0SVersion string }{ - Address: workerIP, - SSHKey: base64.StdEncoding.EncodeToString(s.privateKey), + Address: workerIP, + SSHKey: base64.StdEncoding.EncodeToString(s.privateKey), + K0SVersion: k0sVersion, }) s.Require().NoError(err) - bytes := clusterYaml.Bytes() + bytes := clusterYamlBuf.Bytes() // s.T().Logf("cluster yaml: %s", string(bytes)) resources, err := util.ParseManifests(bytes) s.Require().NoError(err) @@ -274,7 +278,7 @@ metadata: name: remote-test namespace: default spec: - version: v1.27.2-k0s.0 + version: {{ .K0SVersion }}-k0s.0 persistence: type: emptyDir service: @@ -310,7 +314,7 @@ metadata: name: remote-test-0 namespace: default spec: - version: v1.27.2+k0s.0 + version: {{ .K0SVersion }}+k0s.0 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: RemoteMachine