integ-tests: use IAMv3 API key (#13)

We use an IAMv3 key to create the test cluster and the key for the CSI
secret. After each test we clean up the role and key that was created,
if the CSI is torn down.

We also extend the timeout of the tests so that it has enough time to
create the cluster if it does not exist yet.

To make the tests more reliable we wait for Calico and the CSI workloads
to become ready and reduce the amount of requests to avoid rate
limiting.

.github/workflows/integ-tests.yml

@@ -49,5 +49,5 @@ jobs:
             --zone ch-gva-2
         shell: bash
         env:
-          EXOSCALE_API_KEY: ${{ secrets.EXOSCALE_API_KEY }}
-          EXOSCALE_API_SECRET: ${{ secrets.EXOSCALE_API_SECRET }}
+          EXOSCALE_API_KEY: ${{ secrets.EXOSCALE_API_KEY_IAMV3 }}
+          EXOSCALE_API_SECRET: ${{ secrets.EXOSCALE_API_SECRET_IAMV3 }}

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Improvements
+
+* integ-tests: use IAMv3 API key #13 
+
 ## 0.29.2
 
 ### Bug fixes

internal/integ/cluster/setup.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"time"
 
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/flags"
 
@@ -145,15 +144,21 @@ func Setup() error {
 		return err
 	}
 
+	calicoControllerName := "calico-kube-controllers"
+	if err := testCluster.awaitDeploymentReadiness(calicoControllerName); err != nil {
+		slog.Warn("error while awaiting", "deployment", calicoControllerName, "error", err)
+	}
+
+	calicoNodeName := "calico-node"
+	if err := testCluster.awaitDaemonSetReadiness(calicoNodeName); err != nil {
+		slog.Warn("error while awaiting", "DaemonSet", calicoNodeName, "error", err)
+	}
+
 	if !*flags.DontApplyCSI {
 		if err := testCluster.applyCSI(); err != nil {
 			return fmt.Errorf("error applying CSI: %w", err)
 		}
 	}
 
-	// give the CSI some time to become ready
-	// TODO find a more appropriate way to do this.
-	time.Sleep(30 * time.Second)
-
 	return nil
 }

internal/integ/cluster/teardown.go

@@ -51,14 +51,18 @@ func (c *Cluster) tearDownCSI() error {
 	for _, manifestPath := range allManifests {
 		err := c.K8s.DeleteManifest(c.exoV2Context, manifestDir+manifestPath)
 		if err != nil {
-			slog.Error("failed to delete manifest", "manifest", manifestPath)
+			slog.Error("failed to delete manifest", "manifest", manifestPath, "err", err)
 
-			finalErr = fmt.Errorf("errors while deleting manifests")
+			finalErr = fmt.Errorf("errors while deleting manifests: %w", err)
 		}
 	}
 
-	return finalErr
+	err := c.deleteAPIKeyAndRole()
+	if err != nil {
+		slog.Error("failed to clean up CSI API key and role", "err", err)
 
-	// TODO (sauterp) reenable once we have a non-legacy key in GH
-	// return c.deleteAPIKeyAndRole()
+		finalErr = fmt.Errorf("errors while cleaning up CSI API key and role: %w", err)
+	}
+
+	return finalErr
 }

internal/integ/cluster/utils.go

@@ -276,18 +276,87 @@ func (c *Cluster) applyCSI() error {
 	// the CSI controller needs to restart to pick up the new secrets
 	c.restartCSIController()
 
+	controllerName := "exoscale-csi-controller"
+	if err := c.awaitDeploymentReadiness(controllerName); err != nil {
+		slog.Warn("error while awaiting", "deployment", controllerName, "error", err)
+	}
+
+	nodeDriverName := "exoscale-csi-node"
+	if err := c.awaitDaemonSetReadiness(nodeDriverName); err != nil {
+		slog.Warn("error while awaiting", "DaemonSet", nodeDriverName, "error", err)
+	}
+
 	return nil
 }
 
+func retry(trial func() error, nRetries int, retryInterval time.Duration) error {
+	if nRetries == 0 {
+		nRetries = 10
+	}
+
+	if retryInterval == 0 {
+		retryInterval = 10 * time.Second
+	}
+
+	for i := 0; i < nRetries-1; i++ {
+		if trial() == nil {
+			return nil
+		}
+
+		time.Sleep(retryInterval)
+	}
+
+	return trial()
+}
+
+func (c *Cluster) awaitDeploymentReadiness(deploymentName string) error {
+	return retry(func() error {
+		deployment, err := c.K8s.ClientSet.AppsV1().Deployments(csiNamespace).Get(c.exoV2Context, deploymentName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		// check if deployment is ready
+		if deployment.Status.ReadyReplicas == *deployment.Spec.Replicas {
+			slog.Info("ready", "deployment", deploymentName)
+
+			return nil
+		}
+
+		slog.Info("waiting for deployment to become ready", "deployment", deploymentName)
+		return fmt.Errorf("waiting for deployment %q to become ready", deploymentName)
+	}, 0, 0)
+}
+
+func (c *Cluster) awaitDaemonSetReadiness(name string) error {
+	return retry(func() error {
+		daemonSet, err := c.K8s.ClientSet.AppsV1().DaemonSets(csiNamespace).Get(c.exoV2Context, name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		// check if DaemonSet is ready
+		if daemonSet.Status.DesiredNumberScheduled == daemonSet.Status.CurrentNumberScheduled {
+			slog.Info("ready", "DaemonSet", name)
+
+			return nil
+		}
+
+		slog.Info("waiting for DaemonSet to become ready", "DaemonSet", name)
+		return fmt.Errorf("waiting for DaemonSet %q to become ready", name)
+	}, 0, 0)
+}
+
 func (c *Cluster) restartCSIController() {
+	deploymentName := "exoscale-csi-controller"
 	podsClient := c.K8s.ClientSet.CoreV1().Pods(csiNamespace)
 	pods, err := podsClient.List(c.exoV2Context, metav1.ListOptions{})
 	if err != nil {
 		slog.Warn("failed to list pods", "err", err)
 	}
 
 	for _, pod := range pods.Items {
-		if strings.HasPrefix(pod.Name, "exoscale-csi-controller") {
+		if strings.HasPrefix(pod.Name, deploymentName) {
 			err := podsClient.Delete(c.exoV2Context, pod.Name, metav1.DeleteOptions{})
 			if err != nil {
 				slog.Warn("failed to delete pod", "name", pod.Name, "err", err)

internal/integ/integ_test.go

@@ -1,6 +1,7 @@
 package integ
 
 import (
+	"flag"
 	"fmt"
 	"log/slog"
 	"os"
@@ -17,9 +18,17 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	// cluster creation takes a while so we increase the test timeout
+	// This call has to happen before testing.M.Run as that's where
+	// the flag `test.timeout` is used.
+	err := flag.Set("test.timeout", "30m")
+	if err != nil {
+		slog.Warn("failed to set test timeout", "error", err)
+	}
+
 	exitCode := 0
 
-	err := cluster.Setup()
+	err = cluster.Setup()
 	if err != nil {
 		slog.Error("error during setup", "err", err)
 		exitCode = 1

internal/integ/k8s/k8s.go

@@ -44,6 +44,11 @@ func CreateClients(kubeconfig []byte) (*K8S, error) {
 		return nil, err
 	}
 
+	// we frequently encounter rate limiting from the test cluster
+	// to avoid this we set low values.
+	config.QPS = 1
+	config.Burst = 2
+
 	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
 	if err != nil {
 		return nil, err