integ-test: verify that retain policy is respected

deployment/storage-class-retain.yaml

@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: exoscale-bs-retain
+  namespace: kube-system
+provisioner: csi.exoscale.com
+reclaimPolicy: Retain
+allowVolumeExpansion: true

internal/integ/client/client.go

@@ -0,0 +1,18 @@
+package client
+
+import (
+	"fmt"
+
+	"github.com/exoscale/egoscale/v3/credentials"
+
+	exov3 "github.com/exoscale/egoscale/v3"
+)
+
+func CreateEgoscaleClient() (*exov3.Client, error) {
+	v3Client, err := exov3.NewClient(credentials.NewEnvCredentials(), exov3.ClientOptWithEndpoint(exov3.CHGva2))
+	if err != nil {
+		return nil, fmt.Errorf("error setting up egoscale client: %w", err)
+	}
+
+	return v3Client, nil
+}

internal/integ/cluster/setup.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 	"os"
 
+	"github.com/exoscale/exoscale/csi-driver/internal/integ/client"
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/flags"
 
 	exov3 "github.com/exoscale/egoscale/v3"
@@ -101,7 +102,7 @@ func exitApplication(msg string, err error) {
 }
 
 func ConfigureCluster(ctx context.Context, createCluster bool, name, zone string) (*Cluster, error) {
-	client, err := createEgoscaleClient()
+	client, err := client.CreateEgoscaleClient()
 	if err != nil {
 		return nil, fmt.Errorf("error creating egoscale v3 client: %w", err)
 	}

internal/integ/cluster/utils.go

@@ -15,7 +15,6 @@ import (
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/util"
 
 	exov3 "github.com/exoscale/egoscale/v3"
-	"github.com/exoscale/egoscale/v3/credentials"
 )
 
 const (
@@ -33,6 +32,7 @@ var (
 	nodeDriverRBACManifest      = "node-driver-rbac.yaml"
 	nodeDriverManifest          = "node-driver.yaml"
 	storageClassManifest        = "storage-class.yaml"
+	storageClassRetainManifest  = "storage-class-retain.yaml"
 	volumeSnapshotClassManifest = "volume-snapshot-class.yaml"
 
 	allManifests = []string{
@@ -43,6 +43,7 @@ var (
 		nodeDriverRBACManifest,
 		nodeDriverManifest,
 		storageClassManifest,
+		storageClassRetainManifest,
 		volumeSnapshotClassManifest,
 	}
 )
@@ -289,12 +290,3 @@ func (c *Cluster) restartCSIController(ctx context.Context) {
 		}
 	}
 }
-
-func createEgoscaleClient() (*exov3.Client, error) {
-	v3Client, err := exov3.NewClient(credentials.NewEnvCredentials(), exov3.ClientOptWithEndpoint(exov3.CHGva2))
-	if err != nil {
-		return nil, fmt.Errorf("error setting up egoscale client: %w", err)
-	}
-
-	return v3Client, nil
-}

internal/integ/integ_test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"log/slog"
+	"math/rand"
 	"os"
 	"testing"
 	"time"
@@ -13,6 +14,7 @@ import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
+	"github.com/exoscale/exoscale/csi-driver/internal/integ/client"
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/cluster"
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/k8s"
 )
@@ -50,19 +52,9 @@ func TestMain(m *testing.M) {
 	os.Exit(exitCode)
 }
 
-var basicPVC = `
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: my-sbs-pvc
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 100Gi
-  storageClassName: exoscale-sbs
-`
+func generatePVCName(testName string) string {
+	return fmt.Sprintf("%s-%s-%d", "csi-test-pvc", testName, rand.Int())
+}
 
 type getFunc func() interface{}
 
@@ -83,12 +75,14 @@ func awaitExpectation(t *testing.T, expected interface{}, get getFunc) {
 }
 
 func TestVolumeCreation(t *testing.T) {
-	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, "create-vol")
+	testName := "create-vol"
+	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, testName)
 
-	ns.Apply(basicPVC)
+	pvcName := generatePVCName(testName)
+	ns.ApplyPVC(pvcName, false)
 
 	awaitExpectation(t, "Bound", func() interface{} {
-		pvc, err := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name).Get(ns.CTX, "my-sbs-pvc", metav1.GetOptions{})
+		pvc, err := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name).Get(ns.CTX, pvcName, metav1.GetOptions{})
 		assert.NoError(t, err)
 
 		return pvc.Status.Phase
@@ -119,19 +113,21 @@ spec:
       volumes:
         - name: my-awesome-logs
           persistentVolumeClaim:
-            claimName: my-sbs-pvc
+            claimName: %s
 `
 
 func TestVolumeAttach(t *testing.T) {
-	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, "vol-attach")
+	testName := "vol-attach"
+	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, testName)
 
-	ns.Apply(basicPVC)
-	ns.Apply(basicDeployment)
+	pvcName := generatePVCName(testName)
+	ns.ApplyPVC(pvcName, false)
+	ns.Apply(fmt.Sprintf(basicDeployment, pvcName))
 
 	go ns.K.PrintEvents(ns.CTX, ns.Name)
 
 	awaitExpectation(t, "Bound", func() interface{} {
-		pvc, err := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name).Get(ns.CTX, "my-sbs-pvc", metav1.GetOptions{})
+		pvc, err := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name).Get(ns.CTX, pvcName, metav1.GetOptions{})
 		assert.NoError(t, err)
 
 		return pvc.Status.Phase
@@ -150,30 +146,56 @@ func TestVolumeAttach(t *testing.T) {
 }
 
 func TestDeleteVolume(t *testing.T) {
-	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, "del-vol")
+	testName := "del-vol"
+	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, testName)
 
-	ns.Apply(basicPVC)
+	egoClient, err := client.CreateEgoscaleClient()
+	assert.NoError(t, err)
 
-	pvcClient := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name)
+	testFunc := func(useRetainStorageClass bool) func(t *testing.T) {
+		return func(t *testing.T) {
+			pvcName := ""
+			if useRetainStorageClass {
+				pvcName = generatePVCName(testName + "-retain")
+			} else {
+				pvcName = generatePVCName(testName)
+			}
+			ns.ApplyPVC(pvcName, useRetainStorageClass)
 
-	awaitExpectation(t, "Bound", func() interface{} {
-		pvc, err := pvcClient.Get(ns.CTX, "my-sbs-pvc", metav1.GetOptions{})
-		assert.NoError(t, err)
+			pvcClient := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name)
 
-		return pvc.Status.Phase
-	})
+			awaitExpectation(t, "Bound", func() interface{} {
+				pvc, err := pvcClient.Get(ns.CTX, pvcName, metav1.GetOptions{})
+				assert.NoError(t, err)
 
-	err := pvcClient.Delete(ns.CTX, "my-sbs-pvc", metav1.DeleteOptions{})
-	assert.NoError(t, err)
+				return pvc.Status.Phase
+			})
 
-	awaitExpectation(t, 0, func() interface{} {
-		pvcs, err := pvcClient.List(ns.CTX, metav1.ListOptions{})
-		assert.NoError(t, err)
+			err := pvcClient.Delete(ns.CTX, pvcName, metav1.DeleteOptions{})
+			assert.NoError(t, err)
 
-		return len(pvcs.Items)
-	})
+			awaitExpectation(t, 0, func() interface{} {
+				pvcs, err := pvcClient.List(ns.CTX, metav1.ListOptions{})
+				assert.NoError(t, err)
+
+				return len(pvcs.Items)
+			})
+
+			bsVolList, err := egoClient.ListBlockStorageVolumes(ns.CTX)
+			assert.NoError(t, err)
+			for _, volume := range ns.Volumes {
+				_, err := bsVolList.FindBlockStorageVolume(volume)
+				if useRetainStorageClass {
+					assert.NoError(t, err)
+				} else {
+					assert.Error(t, err)
+				}
+			}
+		}
+	}
 
-	// TODO (sauterp) once ego v3 is available check if volume is deleted (and retainPolicy)
+	t.Run("storage-class-delete", testFunc(false))
+	t.Run("storage-class-retain", testFunc(true))
 }
 
 const basicSnapshot = `
@@ -184,7 +206,7 @@ metadata:
 spec:
   volumeSnapshotClassName: exoscale-snapshot
   source:
-    persistentVolumeClaimName: my-sbs-pvc
+    persistentVolumeClaimName: %s
 `
 
 const basicVolumeFromSnapshot = `
@@ -207,21 +229,23 @@ spec:
 `
 
 func TestSnapshot(t *testing.T) {
-	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, "snapshot")
+	testName := "snapshot"
+	ns := k8s.CreateTestNamespace(t, cluster.Get().K8s, testName)
 
-	ns.Apply(basicPVC)
+	pvcName := generatePVCName(testName)
+	ns.ApplyPVC(pvcName, false)
 
 	pvcClient := ns.K.ClientSet.CoreV1().PersistentVolumeClaims(ns.Name)
 
 	awaitExpectation(t, "Bound", func() interface{} {
-		pvc, err := pvcClient.Get(ns.CTX, "my-sbs-pvc", metav1.GetOptions{})
+		pvc, err := pvcClient.Get(ns.CTX, pvcName, metav1.GetOptions{})
 		assert.NoError(t, err)
 
 		return pvc.Status.Phase
 	})
 
 	// create snapshot
-	ns.Apply(basicSnapshot)
+	ns.Apply(fmt.Sprintf(basicSnapshot, pvcName))
 
 	snapshotClient := ns.K.DynamicClient.Resource(getSnapshotCRDResource()).Namespace(ns.Name)
 

internal/integ/k8s/namespace.go

@@ -1,26 +1,75 @@
 package k8s
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"log/slog"
 	"math/rand"
 	"testing"
 
+	"text/template"
+
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 
+	"github.com/exoscale/exoscale/csi-driver/internal/integ/client"
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/flags"
 )
 
 type Namespace struct {
-	K    *K8S
-	t    *testing.T
-	Name string
-	CTX  context.Context
+	K       *K8S
+	t       *testing.T
+	Name    string
+	CTX     context.Context
+	Volumes []string
+}
+
+type PVC struct {
+	Name             string
+	StorageClassName string
+}
+
+var pvcTemplate = `
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Name }}
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: {{ .StorageClassName }}`
+
+func (ns *Namespace) ApplyPVC(name string, useStorageClassRetain bool) {
+	tmpl := template.New("volumeTemplate")
+	parsedTmpl, err := tmpl.Parse(pvcTemplate)
+	if err != nil {
+		slog.Error("failed to parse PVC template", "err", err)
+		return
+	}
+
+	data := PVC{
+		Name:             name,
+		StorageClassName: "exoscale-sbs",
+	}
+	if useStorageClassRetain {
+		data.StorageClassName = "exoscale-bs-retain"
+	}
+	buf := &bytes.Buffer{}
+	if parsedTmpl.Execute(buf, data) != nil {
+		slog.Error("failed to execute PVC template", "err", err)
+		return
+	}
+
+	ns.Apply(buf.String())
+
+	ns.Volumes = append(ns.Volumes, name)
 }
 
 func (ns *Namespace) Apply(manifest string) {
@@ -84,6 +133,19 @@ func CreateTestNamespace(t *testing.T, k *K8S, testName string) *Namespace {
 			slog.Info("cleaning up test namespace", "name", ns.Name)
 			err := ns.K.ClientSet.CoreV1().Namespaces().Delete(ns.CTX, name, metav1.DeleteOptions{})
 			assert.NoError(ns.t, err)
+
+			// delete volumes that may have been retained
+			egoClient, err := client.CreateEgoscaleClient()
+			assert.NoError(ns.t, err)
+
+			bsVolList, err := egoClient.ListBlockStorageVolumes(ns.CTX)
+			assert.NoError(t, err)
+			for _, volume := range ns.Volumes {
+				bsVol, err := bsVolList.FindBlockStorageVolume(volume)
+				if err == nil {
+					egoClient.DeleteBlockStorageVolume(ns.CTX, bsVol.ID)
+				}
+			}
 		})
 	}