From 656d81494162af8b696e1c782b5716dfe308575c Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 23 Oct 2024 13:39:19 +0530 Subject: [PATCH 1/4] ci: script to deploy ceph-csi via ceph-csi-operator Signed-off-by: Praveen M --- scripts/deploy-ceph-csi-operator.sh | 140 ++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 scripts/deploy-ceph-csi-operator.sh diff --git a/scripts/deploy-ceph-csi-operator.sh b/scripts/deploy-ceph-csi-operator.sh new file mode 100644 index 00000000000..5c917355b20 --- /dev/null +++ b/scripts/deploy-ceph-csi-operator.sh @@ -0,0 +1,140 @@ +#!/bin/bash -E + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck disable=SC1091 +[ ! -e "${SCRIPT_DIR}"/utils.sh ] || source "${SCRIPT_DIR}"/utils.sh + +OPERATOR_VERSION=${OPERATOR_VERSION:-"main"} +OPERATOR_URL="https://raw.githubusercontent.com/ceph/ceph-csi-operator/${OPERATOR_VERSION}" + +# operator deployment files +OPERATOR_INSTALL="${OPERATOR_URL}/deploy/all-in-one/install.yaml" + +OPERATOR_NAMESPACE="ceph-csi-operator-system" +IMAGESET_CONFIGMAP_NAME="ceph-csi-imageset" +ENCRYPTION_CONFIGMAP_NAME="ceph-csi-encryption-kms-config" + +# csi drivers +RBD_DRIVER_NAME="rbd.csi.ceph.com" +CEPHFS_DRIVER_NAME="cephfs.csi.ceph.com" +NFS_DRIVER_NAME="nfs.csi.ceph.com" + +# e2e default values +NODE_REGION_LABEL="test.failure-domain/region" +NODE_ZONE_LABEL="test.failure-domain/zone" +CLUSTER_NAME="k8s-cluster-1" + + +TEMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TEMP_DIR"' EXIT + +function create_or_delete_imageset_configmap() { + local operation=$1 + temp_file=$(mktemp "${TEMP_DIR}/imageset-configmap.XXXXXX.yaml") + cat < "${temp_file}" +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${IMAGESET_CONFIGMAP_NAME} + namespace: ${OPERATOR_NAMESPACE} +data: + "plugin": "quay.io/cephcsi/cephcsi:canary" # test image +EOF + kubectl_retry "$operation" -f "${temp_file}" +} + +function create_or_delete_encryption_configmap() { + local operation=$1 + temp_file=$(mktemp "${TEMP_DIR}/encryption-configmap.XXXXXX.yaml") + cat < "${temp_file}" +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: ${OPERATOR_NAMESPACE} + name: ${ENCRYPTION_CONFIGMAP_NAME} +data: + config.json: "" +EOF + kubectl_retry "$operation" -f "${temp_file}" +} + +function create_or_delete_operator_config() { + # TODO: encryption config + local operation=$1 + create_or_delete_imageset_configmap "$operation" + create_or_delete_encryption_configmap "$operation" + + temp_file=$(mktemp "${TEMP_DIR}/operatorconfig.XXXXXX.yaml") + cat < "${temp_file}" +apiVersion: csi.ceph.io/v1alpha1 +kind: OperatorConfig +metadata: + name: ceph-csi-operator-config + namespace: ${OPERATOR_NAMESPACE} +spec: + driverSpecDefaults: + nodePlugin: + topology: + domainLabels: + - ${NODE_REGION_LABEL} + - ${NODE_ZONE_LABEL} + generateOMapInfo: true + enableMetadata: true + clusterName: ${CLUSTER_NAME} + log: + verbosity: 5 # csi pods log level + imageSet: + name: ${IMAGESET_CONFIGMAP_NAME} + encryption: + configMapName: + name: ${ENCRYPTION_CONFIGMAP_NAME} + log: + verbosity: 3 # operator log level +EOF + kubectl_retry "$operation" -f "${temp_file}" +} + +function deploy_or_delete_driver() { + local operator=$1 + local driver_name=$2 + temp_file=$(mktemp "${TEMP_DIR}/${driver_name}.XXXXXX.yaml") + cat < "${temp_file}" +apiVersion: csi.ceph.io/v1alpha1 +kind: Driver +metadata: + name: ${driver_name} + namespace: ${OPERATOR_NAMESPACE} +EOF + kubectl_retry "$operator" -f "${temp_file}" +} + +function deploy_operator() { + kubectl_retry create -f "${OPERATOR_INSTALL}" + create_or_delete_operator_config "create" + deploy_or_delete_driver "create" $RBD_DRIVER_NAME + deploy_or_delete_driver "create" $CEPHFS_DRIVER_NAME + deploy_or_delete_driver "create" $NFS_DRIVER_NAME +} + +function cleanup() { + deploy_or_delete_driver "delete" $RBD_DRIVER_NAME + deploy_or_delete_driver "delete" $CEPHFS_DRIVER_NAME + deploy_or_delete_driver "delete" $NFS_DRIVER_NAME + create_or_delete_operator_config "delete" + kubectl_retry "delete" -f "${OPERATOR_INSTALL}" +} + +case "${1:-}" in + deploy) + deploy_operator + ;; + cleanup) + cleanup + ;; + *) + echo "Usage:" >&2 + echo " $0 deploy" >&2 + echo " $0 cleanup" >&2 + exit 1 + ;; +esac From 0df5de9d572585469150663c061badf6948fde4b Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 23 Oct 2024 13:40:52 +0530 Subject: [PATCH 2/4] e2e: add operatorDeployment flag This commit adds `operatorDeployment` flag, defaults to false. Set this to true for running test on ceph-csi deployment via operator. Signed-off-by: Praveen M --- e2e/e2e_test.go | 1 + e2e/utils.go | 41 +++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index e4538788aeb..671029be22d 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -52,6 +52,7 @@ func init() { flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use") flag.StringVar(&clusterID, "clusterid", "", "Ceph cluster ID to use (defaults to `ceph fsid` detection)") flag.StringVar(&nfsDriverName, "nfs-driver", "nfs.csi.ceph.com", "name of the driver for NFS-volumes") + flag.BoolVar(&operatorDeployment, "operator-deployment", false, "test running on deployment via operator") setDefaultKubeconfig() // Register framework flags, then handle flags diff --git a/e2e/utils.go b/e2e/utils.go index 0e29f53c168..05ad46013bb 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -80,26 +80,27 @@ const ( var ( // cli flags. - deployTimeout int - deployCephFS bool - deployRBD bool - deployNFS bool - testCephFS bool - testCephFSFscrypt bool - testRBD bool - testRBDFSCrypt bool - testNBD bool - testNFS bool - helmTest bool - upgradeTesting bool - upgradeVersion string - cephCSINamespace string - rookNamespace string - radosNamespace string - poll = 2 * time.Second - isOpenShift bool - clusterID string - nfsDriverName string + deployTimeout int + deployCephFS bool + deployRBD bool + deployNFS bool + testCephFS bool + testCephFSFscrypt bool + testRBD bool + testRBDFSCrypt bool + testNBD bool + testNFS bool + helmTest bool + upgradeTesting bool + upgradeVersion string + cephCSINamespace string + rookNamespace string + radosNamespace string + poll = 2 * time.Second + isOpenShift bool + clusterID string + nfsDriverName string + operatorDeployment bool ) type cephfsFilesystem struct { From 4a34a891cfb8ee07235c4858b8eb41756f4e3326 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 23 Oct 2024 13:44:50 +0530 Subject: [PATCH 3/4] e2e: unset readAffinity cli option Signed-off-by: Praveen M --- e2e/deployment.go | 12 ------------ e2e/rbd.go | 2 -- 2 files changed, 14 deletions(-) diff --git a/e2e/deployment.go b/e2e/deployment.go index 12b86acdacc..f70ee3cd6a8 100644 --- a/e2e/deployment.go +++ b/e2e/deployment.go @@ -235,13 +235,9 @@ type yamlResourceNamespaced struct { filename string namespace string domainLabel string - crushLocationLabels string // set the number of replicas in a Deployment to 1. oneReplica bool - - // enable read affinity support (for RBD) - enableReadAffinity bool } func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error { @@ -260,14 +256,6 @@ func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error { data = addTopologyDomainsToDSYaml(data, yrn.domainLabel) } - if yrn.enableReadAffinity { - data = enableReadAffinityInTemplate(data) - } - - if yrn.crushLocationLabels != "" { - data = addCrsuhLocationLabels(data, yrn.crushLocationLabels) - } - err = retryKubectlInput(yrn.namespace, action, data, deployTimeout) if err != nil { return fmt.Errorf("failed to %s resource %q in namespace %q: %w", action, yrn.filename, yrn.namespace, err) diff --git a/e2e/rbd.go b/e2e/rbd.go index c44a6441afa..619f7d36856 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -170,8 +170,6 @@ func createORDeleteRbdResources(action kubectlAction) { filename: rbdDirPath + rbdNodePlugin, namespace: cephCSINamespace, domainLabel: nodeRegionLabel + "," + nodeZoneLabel, - enableReadAffinity: true, - crushLocationLabels: crushLocationRegionLabel + "," + crushLocationZoneLabel, }, } From 0013f7ab9356f3853fb07ca6d569a3c26204b0f5 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 6 Nov 2024 17:27:50 +0530 Subject: [PATCH 4/4] e2e: handle ceph-csi-operator deployment changes This commits adds e2e/operator.go containing utility methods specific to the operator. Signed-off-by: Praveen M --- e2e/cephfs.go | 20 ++++++++++++---- e2e/nfs.go | 7 ++++++ e2e/operator.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ e2e/rbd.go | 57 +++++++++++++++++++++++++++++++-------------- e2e/utils.go | 2 ++ 5 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 e2e/operator.go diff --git a/e2e/cephfs.go b/e2e/cephfs.go index c6050441b93..b84282c1a4e 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -46,6 +46,9 @@ var ( subvolumegroup = "e2e" fileSystemName = "myfs" fileSystemPoolName = "myfs-replicated" + + operatorCephFSDeploymentName = "cephfs.csi.ceph.com-ctrlplugin" + operatorCephFSDaemonsetName = "cephfs.csi.ceph.com-nodeplugin" ) func deployCephfsPlugin() { @@ -175,6 +178,11 @@ var _ = Describe(cephfsType, func() { Skip("Skipping CephFS E2E") } c = f.ClientSet + if operatorDeployment { + cephFSDeploymentName = operatorCephFSDeploymentName + cephFSDeamonSetName = operatorCephFSDaemonsetName + } + if deployCephFS { if cephCSINamespace != defaultNs { err := createNamespace(c, cephCSINamespace) @@ -209,11 +217,15 @@ var _ = Describe(cephfsType, func() { deployVault(f.ClientSet, deployTimeout) // wait for cluster name update in deployment - containers := []string{cephFSContainerName} - err = waitForContainersArgsUpdate(c, cephCSINamespace, cephFSDeploymentName, - "clustername", defaultClusterName, containers, deployTimeout) + if operatorDeployment { + err = setClusterName(defaultClusterName) + } else { + containers := []string{cephFSContainerName} + err = waitForContainersArgsUpdate(c, cephCSINamespace, cephFSDeploymentName, + "clustername", defaultClusterName, containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, cephFSDeploymentName, err) + framework.Failf("timeout waiting for clustername arg update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } err = createSubvolumegroup(f, fileSystemName, subvolumegroup) diff --git a/e2e/nfs.go b/e2e/nfs.go index a85004b50a4..d21814f7e4e 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -50,6 +50,9 @@ var ( // FIXME: some tests change the subvolumegroup to "e2e". defaultSubvolumegroup = "csi" + + operatorNFSDeploymentName = "nfs.csi.ceph.com-ctrlplugin" + operatorNFSDaemonsetName = "nfs.csi.ceph.com-nodeplugin" ) func deployNFSPlugin(f *framework.Framework) { @@ -242,6 +245,10 @@ var _ = Describe("nfs", func() { Skip("Skipping NFS E2E") } c = f.ClientSet + if operatorDeployment { + nfsDeploymentName = operatorNFSDeploymentName + nfsDeamonSetName = operatorNFSDaemonsetName + } if deployNFS { if cephCSINamespace != defaultNs { err := createNamespace(c, cephCSINamespace) diff --git a/e2e/operator.go b/e2e/operator.go new file mode 100644 index 00000000000..71f1319e790 --- /dev/null +++ b/e2e/operator.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" +) + +const ( + OperatorConfigName = "ceph-csi-operator-config" + OperatorNamespace = "ceph-csi-operator-system" +) + +func setEnableMetadata(value bool) error{ + command := []string{ + "operatorconfigs.csi.ceph.io", + OperatorConfigName, + "--type=merge", + "-p", + fmt.Sprintf(`{"spec": {"driverSpecDefaults": {"enableMetadata": %t}}}`, value), + } + + // Patch the operator config + err := retryKubectlArgs(OperatorNamespace, kubectlPatch, deployTimeout, command...) + if err != nil { + return err + } + + return nil +} + +func setClusterName(value string) error { + command := []string{ + "operatorconfigs.csi.ceph.io", + OperatorConfigName, + "--type=merge", + "-p", + fmt.Sprintf(`{"spec": {"driverSpecDefaults": {"clusterName": %s}}}`, value), + } + + // Patch the operator config + err := retryKubectlArgs(OperatorNamespace, kubectlPatch, deployTimeout, command...) + if err != nil { + return err + } + + return nil +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 619f7d36856..318922c4df2 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -108,6 +108,10 @@ var ( volSnapNameKey = "csi.storage.k8s.io/volumesnapshot/name" volSnapNamespaceKey = "csi.storage.k8s.io/volumesnapshot/namespace" volSnapContentNameKey = "csi.storage.k8s.io/volumesnapshotcontent/name" + + operatorRBDDeploymentName = "rbd.csi.ceph.com-ctrlplugin" + operatorRBDDaemonsetName = "rbd.csi.ceph.com-nodeplugin" + rbdPodSelector = fmt.Sprintf("app in (ceph-csi-rbd, %s, %s, %s, %s)", rbdDeploymentName, rbdDaemonsetName, operatorRBDDeploymentName, operatorRBDDaemonsetName) ) func deployRBDPlugin() { @@ -167,9 +171,9 @@ func createORDeleteRbdResources(action kubectlAction) { }, // the node-plugin itself &yamlResourceNamespaced{ - filename: rbdDirPath + rbdNodePlugin, - namespace: cephCSINamespace, - domainLabel: nodeRegionLabel + "," + nodeZoneLabel, + filename: rbdDirPath + rbdNodePlugin, + namespace: cephCSINamespace, + domainLabel: nodeRegionLabel + "," + nodeZoneLabel, }, } @@ -272,6 +276,10 @@ var _ = Describe("RBD", func() { Skip("Skipping RBD E2E") } c = f.ClientSet + if operatorDeployment { + rbdDeploymentName = operatorRBDDeploymentName + rbdDaemonsetName = operatorRBDDaemonsetName + } if deployRBD { err := addLabelsToNodes(f, map[string]string{ nodeRegionLabel: regionValue, @@ -344,11 +352,15 @@ var _ = Describe("RBD", func() { } // wait for cluster name update in deployment - containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "clustername", defaultClusterName, containers, deployTimeout) + if operatorDeployment { + err = setClusterName(defaultClusterName) + } else { + containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "clustername", defaultClusterName, containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("timeout waiting for clustername arg update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } }) @@ -2814,7 +2826,11 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 1, defaultRBDPool) validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType) // delete rbd nodeplugin pods - err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false) + selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName) + if err != nil { + framework.Failf("failed to get the labels: %v", err) + } + err = deletePodWithLabel(selector, cephCSINamespace, false) if err != nil { framework.Failf("fail to delete pod: %v", err) } @@ -3781,8 +3797,7 @@ var _ = Describe("RBD", func() { framework.Failf("failed to create rados namespace: %v", err) } // delete csi pods - err = deletePodWithLabel("app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)", - cephCSINamespace, false) + err = deletePodWithLabel(rbdPodSelector, cephCSINamespace, false) if err != nil { framework.Failf("failed to delete pods with labels: %v", err) } @@ -4600,10 +4615,14 @@ var _ = Describe("RBD", func() { // wait for cluster name update in deployment containers := []string{"csi-rbdplugin", "csi-rbdplugin-controller"} - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "setmetadata", "false", containers, deployTimeout) + if operatorDeployment { + err = setEnableMetadata(false) + } else { + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "setmetadata", "false", containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("failed to update setmetadata arg in %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } pvcSmartClone, err := loadPVC(pvcSmartClonePath) if err != nil { @@ -4703,11 +4722,15 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0, defaultRBDPool) validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType) validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType) - // wait for cluster name update in deployment - err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, - "setmetadata", "true", containers, deployTimeout) + if operatorDeployment { + err = setEnableMetadata(true) + } else { + // wait for cluster name update in deployment + err = waitForContainersArgsUpdate(c, cephCSINamespace, rbdDeploymentName, + "setmetadata", "true", containers, deployTimeout) + } if err != nil { - framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) + framework.Failf("failed to update setmetadata arg in %s/%s: %v", cephCSINamespace, rbdDeploymentName, err) } }) diff --git a/e2e/utils.go b/e2e/utils.go index 05ad46013bb..bee1400e9c5 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1621,6 +1621,8 @@ const ( kubectlCreate = kubectlAction("create") // kubectlDelete tells retryKubectlInput() to run "delete". kubectlDelete = kubectlAction("delete") + // kubectlPatch tells retryKubectlInput() to run "patch". + kubectlPatch = kubectlAction("patch") ) // String returns the string format of the kubectlAction, this is automatically