Skip to content

Commit

Permalink
rbd: support QoS based on capacity for rbd volume
Browse files Browse the repository at this point in the history 
1. QoS provides settings for rbd volume read/write iops
   and read/write bandwidth.
2. All QoS parameters are placed in the SC,
   send QoS parameters from SC to Cephcsi through PVC create request.
3. We need provide QoS parameters in the SC as below:
   - BaseReadIops
   - BaseWriteIops
   - BaseReadBytesPerSecond
   - BaseWriteBytesPerSecond
   - ReadIopsPerGB
   - WriteIopsPerGB
   - ReadBpsPerGB
   - WriteBpsPerGB
   - BaseVolSizeBytes
   There are 4 base qos parameters among them, when users apply for
   a volume capacity equal to or less than BaseVolSizebytes, use base
   qos limit. For the portion of capacity exceeding BaseVolSizebytes,
   QoS will be increased in steps set per GB. If the step size parameter
   per GB is not provided, only base QoS limit will be used and not associated
   with capacity size.
4. If PVC has resize request, adjust the QoS limit
   according to the QoS parameters after resizing.

Signed-off-by: Yite Gu <[email protected]>
  • Loading branch information
@YiteGu
YiteGu committed Dec 19, 2024
1 parent 3c63fea commit 47bd9ab
Show file tree
Hide file tree
Showing 8 changed files with 586 additions and 1 deletion.
9 changes: 9 additions & 0 deletions docs/rbd/deploy.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ make image-cephcsi
| `stripeUnit` | no | stripe unit in bytes |
| `stripeCount` | no | objects to stripe over before looping |
| `objectSize` | no | object size in bytes |
| `BaseReadIops` | no | the base limit of read operations per second |
| `BaseWriteIops` | no | the base limit of write operations per second |
| `BaseReadBytesPerSecond` | no | the base limit of read bytes per second |
| `BaseWriteBytesPerSecond` | no | the base limit of write bytes per second |
| `ReadIopsPerGB` | no | the limit of read operations per GiB |
| `WriteIopsPerGB` | no | the limit of write operations per GiB |
| `ReadBpsPerGB` | no | the limit of read bytes per GiB |
| `WriteBpsPerGB` | no | the limit of write bytes per GiB |
| `BaseVolSizeBytes` | no | the min size of volume what use to calc qos beased on capacity |
| `extraDeploy` | no | array of extra objects to deploy with the release |

**NOTE:** An accompanying CSI configuration file, needs to be provided to the
Expand Down
59 changes: 59 additions & 0 deletions e2e/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4643,6 +4643,65 @@ var _ = Describe("RBD", func() {
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})

By("validate rbd image qos", func() {
qosParameters := map[string]string{
"BaseReadIops": "2000",
"BaseWriteIops": "1000",
"BaseReadBytesPerSecond": "209715200",
"baseWriteBytesPerSecond": "104857600",
}
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}

err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
}()

// create PVC
pvc, err := loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}

pvc.Namespace = f.UniqueName

err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}

// validate rbd image qos
err = validateQOS(f, pvc, qosParameters)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}

// delete pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
})

By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() {
err := createRBDSnapshotClass(f)
if err != nil {
Expand Down
33 changes: 33 additions & 0 deletions e2e/rbd_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1166,3 +1166,36 @@ func validateStripe(f *framework.Framework,

return nil
}

func validateQOS(f *framework.Framework,
pvc *v1.PersistentVolumeClaim,
qosParameters map[string]string,
) error {
imageQosPair := map[string]string{
"BaseReadIops": "rbd_qos_read_iops_limit",
"BaseWriteIops": "rbd_qos_write_iops_limit",
"BaseReadBytesPerSecond": "rbd_qos_read_bps_limit",
"BaseWriteBytesPerSecond": "rbd_qos_write_bps_limit",
}
metadataConfPrefix := "conf_"

imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}

rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
for k, v := range qosParameters {
if r, ok := imageQosPair[k]; ok {
qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+r, f)
if err != nil {
return err
}
if qosVal != v {
return fmt.Errorf("%s: %s does not match expected %s", r, qosVal, v)
}
}
}

return nil
}
32 changes: 32 additions & 0 deletions examples/rbd/storageclass.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,38 @@ parameters:
# stripeCount: <>
# (optional) The object size in bytes.
# objectSize: <>

# rbd volume QoS.
# QoS provides settings for rbd volume read/write iops
# and read/write bandwidth. There are 4 base qos parameters
# among them, when users apply for a volume capacity equal
# to or less than BaseVolSizebytes, use base qos limit.
# For the portion of capacity exceeding BaseVolSizebytes,
# QoS will be increased in steps set per GiB. If the step
# size parameter per GiB is not provided, only base QoS limit
# will be used and not associated with capacity size.
#
# note: currently supports rbd-nbd mounter.
#
# For more details
# (optional) the base limit of read operations per second.
# BaseReadIops: <>
# (optional) the base limit of write operations per second.
# BaseWriteIops: <>
# (optional) the base limit of read bytes per second.
# BaseReadBytesPerSecond: <>
# (optional) the base limit of write bytes per second.
# BaseWriteBytesPerSecond: <>
# (optional) the limit of read operations per GiB.
# ReadIopsPerGB: <>
# (optional) the limit of write operations per GiB.
# WriteIopsPerGB: <>
# (optional) the limit of read bytes per GiB.
# ReadBpsPerGB: <>
# (optional) the limit of write bytes per GiB.
# WriteBpsPerGB: <>
# (optional) min size of volume what use to calc qos beased on capacity.
# BaseVolSizeBytes:<>
reclaimPolicy: Delete
allowVolumeExpansion: true

Expand Down
31 changes: 30 additions & 1 deletion internal/rbd/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error())
}

// Get QosParameters from SC if qos configuration existing in SC
err = rbdVol.SetQOS(ctx, req.GetParameters())
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}

err = rbdVol.Connect(cr)
if err != nil {
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
Expand Down Expand Up @@ -415,7 +421,7 @@ func (cs *ControllerServer) CreateVolume(
}
}()

err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap)
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap, req.GetParameters())
if err != nil {
if errors.Is(err, ErrFlattenInProgress) {
return nil, status.Error(codes.Aborted, err.Error())
Expand Down Expand Up @@ -712,6 +718,7 @@ func (cs *ControllerServer) createBackingImage(
secrets map[string]string,
rbdVol, parentVol *rbdVolume,
rbdSnap *rbdSnapshot,
scParams map[string]string,
) error {
var err error

Expand Down Expand Up @@ -766,6 +773,21 @@ func (cs *ControllerServer) createBackingImage(
return status.Error(codes.Internal, err.Error())
}

// Apply Qos parameters to rbd image.
err = rbdVol.ApplyQOS(ctx)
if err != nil {
log.DebugLog(ctx, "failed apply QOS for rbd image: %v", err)

return status.Error(codes.Internal, err.Error())
}
// Save Qos parameters from SC in Image medatate, we will use it while resize volume.
err = rbdVol.SaveQOS(ctx, scParams)
if err != nil {
log.DebugLog(ctx, "failed save QOS for rbd image: %v", err)

return status.Error(codes.Internal, err.Error())
}

return nil
}

Expand Down Expand Up @@ -1605,6 +1627,13 @@ func (cs *ControllerServer) ControllerExpandVolume(
if err != nil {
log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err)

return nil, status.Error(codes.Internal, err.Error())
}
// adjust rbd qos after resize volume.
err = rbdVol.AdjustQOS(ctx)
if err != nil {
log.DebugLog(ctx, "failed adjust QOS for rbd image")

return nil, status.Error(codes.Internal, err.Error())
}
}
Expand Down
Loading

0 comments on commit 47bd9ab

Please sign in to comment.