diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 05bf0beab5b0..b24c6b2e128a 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -28,6 +28,11 @@ serviceAccounts: # cephFS: # subvolumeGroup: "csi" # netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" +# readAffinity: +# enabled: true +# crushLocationLabels: +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone csiConfig: [] # Labels to apply to all resources diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 0d43b1671109..64b74e2abd2d 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -27,6 +27,11 @@ serviceAccounts: # - "" # rbd: # netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" +# readAffinity: +# enabled: true +# crushLocationLabels: +# - topology.kubernetes.io/region +# - topology.kubernetes.io/zone csiConfig: [] # Configuration details of clusterID,PoolID and FscID mapping diff --git a/deploy/csi-config-map-sample.yaml b/deploy/csi-config-map-sample.yaml index b48e834a56f6..6397bbc84855 100644 --- a/deploy/csi-config-map-sample.yaml +++ b/deploy/csi-config-map-sample.yaml @@ -66,6 +66,15 @@ data: } "nfs": { "netNamespaceFilePath": "/plugins/nfs.csi.ceph.com/net", + }, + "readAffinity": { + "enabled": "false", + "crushLocationLabels": [ + "", + "" + ... + "" + ] } } ] diff --git a/internal/csi-common/driver.go b/internal/csi-common/driver.go index 31c89070e089..8841f99bb61e 100644 --- a/internal/csi-common/driver.go +++ b/internal/csi-common/driver.go @@ -28,7 +28,7 @@ import ( // CSIDriver stores driver information. type CSIDriver struct { name string - nodeID string + NodeID string version string // topology constraints that this nodeserver will advertise topology map[string]string @@ -61,7 +61,7 @@ func NewCSIDriver(name, v, nodeID string) *CSIDriver { driver := CSIDriver{ name: name, version: v, - nodeID: nodeID, + NodeID: nodeID, } return &driver diff --git a/internal/csi-common/nodeserver-default.go b/internal/csi-common/nodeserver-default.go index 40206980de60..ce1c65fe7509 100644 --- a/internal/csi-common/nodeserver-default.go +++ b/internal/csi-common/nodeserver-default.go @@ -55,7 +55,7 @@ func (ns *DefaultNodeServer) NodeGetInfo( } return &csi.NodeGetInfoResponse{ - NodeId: ns.Driver.nodeID, + NodeId: ns.Driver.NodeID, AccessibleTopology: csiTopology, }, nil } diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 25f51b35c125..a4db841b7fc2 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -280,14 +280,23 @@ func (ns *NodeServer) populateRbdVol( // appendReadAffinityMapOptions appends readAffinityMapOptions to mapOptions // if mounter is rbdDefaultMounter and readAffinityMapOptions is not empty. +// readAffinityMapOptions is read from csi config file if present else read +// from the command line argument. func (ns NodeServer) appendReadAffinityMapOptions(rv *rbdVolume) { + + var readAffinityMapOptions string + readAffinityMapOptions = ns.getReadAffinityMapOptionFromConfigMap(rv) + if readAffinityMapOptions == "" { + readAffinityMapOptions = ns.readAffinityMapOptions + } + switch { - case ns.readAffinityMapOptions == "" || rv.Mounter != rbdDefaultMounter: + case readAffinityMapOptions == "" || rv.Mounter != rbdDefaultMounter: return case rv.MapOptions != "": - rv.MapOptions += "," + ns.readAffinityMapOptions + rv.MapOptions += "," + readAffinityMapOptions default: - rv.MapOptions = ns.readAffinityMapOptions + rv.MapOptions = readAffinityMapOptions } } @@ -1397,8 +1406,28 @@ func getDeviceSize(ctx context.Context, devicePath string) (uint64, error) { } func (ns *NodeServer) SetReadAffinityMapOptions(crushLocationMap map[string]string) { + ns.readAffinityMapOptions = constructReadAffinityMapOption(crushLocationMap) +} + +func (ns *NodeServer) getReadAffinityMapOptionFromConfigMap(rv *rbdVolume) string { + crushLocationLabels, err := util.GetReadAffinityOptions(util.CsiConfigFile, rv.ClusterID) + if err != nil { + log.FatalLogMsg(err.Error()) + } + + crushLocationMap, err := util.GetCrushLocationMap(crushLocationLabels, ns.Driver.NodeID) + if err != nil { + log.FatalLogMsg(err.Error()) + } + + readAffinityMapOptions := constructReadAffinityMapOption(crushLocationMap) + + return readAffinityMapOptions +} + +func constructReadAffinityMapOption(crushLocationMap map[string]string) string { if len(crushLocationMap) == 0 { - return + return "" } var b strings.Builder @@ -1412,5 +1441,6 @@ func (ns *NodeServer) SetReadAffinityMapOptions(crushLocationMap map[string]stri b.WriteString(fmt.Sprintf("|%s:%s", key, val)) } } - ns.readAffinityMapOptions = b.String() + + return b.String() } diff --git a/internal/util/csiconfig.go b/internal/util/csiconfig.go index 48a2e09c1fe4..e9aa1b4520a2 100644 --- a/internal/util/csiconfig.go +++ b/internal/util/csiconfig.go @@ -64,6 +64,11 @@ type ClusterInfo struct { // symlink filepath for the network namespace where we need to execute commands. NetNamespaceFilePath string `json:"netNamespaceFilePath"` } `json:"nfs"` + // Read affinity map options + ReadAffinity struct { + Enabled string `json:"enabled"` + CrushLocationLabels []string `json:"crushLocationLabels"` + } } // Expected JSON structure in the passed in config file is, @@ -209,3 +214,24 @@ func GetNFSNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) return cluster.NFS.NetNamespaceFilePath, nil } + +// GetReadAffinityOptions returns the crushLocationLabels from csi config for the given clusterID +// If `readAffinity.enabled` is set to true +func GetReadAffinityOptions(pathToConfig, clusterId string) (string, error) { + cluster, err := readClusterInfo(pathToConfig, clusterId) + if err != nil { + return "", err + } + + if cluster.ReadAffinity.Enabled == "false" { + return "", nil + } + + if len(cluster.ReadAffinity.CrushLocationLabels) == 0 { + return "", fmt.Errorf("empty crush loction labels list for cluster ID (%s) in config", clusterId) + } + + crushLocationLabels := strings.Join(cluster.ReadAffinity.CrushLocationLabels, ",") + + return crushLocationLabels, nil +}