From 3c13f8231ea3f8efd47b53dac2a41d389dc3f3ba Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 30 Oct 2024 09:47:04 +0100 Subject: [PATCH 01/16] rbd: close the RBD-image after adding it to a VolumeGroup When the image is not closed, it keeps a watch open. This prevents the CSI Controller to delete the Volume, as there is still a user of it. Fixes: f9ab14e826 "rbd: check if an image is part of a group before adding it" Signed-off-by: Niels de Vos --- internal/rbd/group.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/rbd/group.go b/internal/rbd/group.go index d0a39032220..701e3b47d63 100644 --- a/internal/rbd/group.go +++ b/internal/rbd/group.go @@ -44,6 +44,7 @@ func (rv *rbdVolume) AddToGroup(ctx context.Context, vg types.VolumeGroup) error if err != nil { return fmt.Errorf("failed to open image %q: %w", rv, err) } + defer image.Close() info, err := image.GetGroup() if err != nil { From b1a9498358005f75449f27aa7fa078961ecd4f2d Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 31 Oct 2024 17:07:33 +0100 Subject: [PATCH 02/16] journal: store `csi.groupid` for snapshots Commit 95733b3a9 introduced the `StoreGroupID()` function, but that unfortunately set an empty key in the journal. Passing the `csiGroupIDKey` key (with value `csi.groupid`) caused setting `csi.csi.groupid` as a key. Reading the value back with the right `csi.groupid` key always returned an empty value. Fixes: 95733b3a9 "journal: add option to store the groupID" Signed-off-by: Niels de Vos --- internal/journal/voljournal.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/journal/voljournal.go b/internal/journal/voljournal.go index 096375a68ac..d57ece947f4 100644 --- a/internal/journal/voljournal.go +++ b/internal/journal/voljournal.go @@ -199,6 +199,7 @@ func NewCSISnapshotJournal(suffix string) *Config { cephSnapSourceKey: "csi.source", namespace: "", csiImageIDKey: "csi.imageid", + csiGroupIDKey: "csi.groupid", encryptKMSKey: "csi.volume.encryptKMS", encryptionType: "csi.volume.encryptionType", ownerKey: "csi.volume.owner", @@ -805,7 +806,8 @@ func (conn *Connection) StoreAttribute(ctx context.Context, pool, reservedUUID, // StoreGroupID stores an groupID in omap. func (conn *Connection) StoreGroupID(ctx context.Context, pool, reservedUUID, groupID string) error { - err := conn.StoreAttribute(ctx, pool, reservedUUID, conn.config.csiGroupIDKey, groupID) + err := setOMapKeys(ctx, conn, pool, conn.config.namespace, conn.config.cephUUIDDirectoryPrefix+reservedUUID, + map[string]string{conn.config.csiGroupIDKey: groupID}) if err != nil { return fmt.Errorf("failed to store groupID %w", err) } From 34dd709e4ec2f578e7b68b293de20473bb485811 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 1 Oct 2024 10:37:35 +0200 Subject: [PATCH 03/16] rbd: have `GetVolumeGroup()` return an empty volume group if it was not found Signed-off-by: Niels de Vos --- internal/rbd/group/volume_group.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go index bb1247e4c5c..da682435652 100644 --- a/internal/rbd/group/volume_group.go +++ b/internal/rbd/group/volume_group.go @@ -74,6 +74,12 @@ func GetVolumeGroup( attrs, err := vg.getVolumeGroupAttributes(ctx) if err != nil { + if errors.Is(err, librbd.ErrNotFound) { + log.ErrorLog(ctx, "%v, returning empty volume group %q", vg, err) + + return vg, err + } + return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vg, err) } From 412892832a6ea0a8feb34bfb0a2b7e0b54ab9ab1 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 25 Sep 2024 16:34:56 +0200 Subject: [PATCH 04/16] rbd: add `.requestName` to the `commonVolumeGroup` struct Signed-off-by: Niels de Vos --- internal/rbd/group/util.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/rbd/group/util.go b/internal/rbd/group/util.go index f32b24cd8ad..a588eba3eb3 100644 --- a/internal/rbd/group/util.go +++ b/internal/rbd/group/util.go @@ -34,6 +34,9 @@ type commonVolumeGroup struct { // is used to find the group in the journal. id string + // requestName is passed by the caller when a group is created. + requestName string + // name is used in RBD API calls as the name of this object name string @@ -130,6 +133,7 @@ func (cvg *commonVolumeGroup) getVolumeGroupAttributes(ctx context.Context) (*jo attrs = &journal.VolumeGroupAttributes{} } + cvg.requestName = attrs.RequestName cvg.name = attrs.GroupName cvg.creationTime = attrs.CreationTime From 84f99da9d9a42b256084e624b0492b79f2b914fc Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 1 Oct 2024 09:35:50 +0200 Subject: [PATCH 05/16] rbd: pass CSI-drivername to volume group instead of journal instance Each object is responsible for maintaining a connection to the journal. By sharing a single journal, cleanup of objects becomes more complex as the journal is used in deferred functions and only the last should destroy the journal connection resources. Signed-off-by: Niels de Vos --- internal/rbd/group/util.go | 58 ++++++++++++++++++++++++++---- internal/rbd/group/volume_group.go | 21 +++++++---- internal/rbd/manager.go | 14 ++------ 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/internal/rbd/group/util.go b/internal/rbd/group/util.go index a588eba3eb3..39583edf027 100644 --- a/internal/rbd/group/util.go +++ b/internal/rbd/group/util.go @@ -57,13 +57,16 @@ type commonVolumeGroup struct { pool string namespace string + // csiDriver is the CSI drivername that is required to connect the journal + csiDriver string + // use getJournal() to make sure the journal is connected journal journal.VolumeGroupJournal } func (cvg *commonVolumeGroup) initCommonVolumeGroup( ctx context.Context, id string, - j journal.VolumeGroupJournal, + csiDriver string, creds *util.Credentials, ) error { csiID := util.CSIIdentifier{} @@ -87,7 +90,7 @@ func (cvg *commonVolumeGroup) initCommonVolumeGroup( return fmt.Errorf("failed to get pool for volume group id %q: %w", id, err) } - cvg.journal = j + cvg.csiDriver = csiDriver cvg.credentials = creds cvg.id = id cvg.clusterID = csiID.ClusterID @@ -113,18 +116,27 @@ func (cvg *commonVolumeGroup) Destroy(ctx context.Context) { } if cvg.credentials != nil { - cvg.credentials.DeleteCredentials() + // credentials should only be removed with DeleteCredentials() + // by the caller that allocated them cvg.credentials = nil } - log.DebugLog(ctx, "destroyed volume group instance with id %q", cvg.id) + if cvg.journal != nil { + cvg.journal.Destroy() + cvg.journal = nil + } } // getVolumeGroupAttributes fetches the attributes from the journal, sets some // of the common values for the VolumeGroup and returns the attributes struct // for further consumption (like checking the VolumeMap). func (cvg *commonVolumeGroup) getVolumeGroupAttributes(ctx context.Context) (*journal.VolumeGroupAttributes, error) { - attrs, err := cvg.journal.GetVolumeGroupAttributes(ctx, cvg.pool, cvg.objectUUID) + j, err := cvg.getJournal(ctx) + if err != nil { + return nil, err + } + + attrs, err := j.GetVolumeGroupAttributes(ctx, cvg.pool, cvg.objectUUID) if err != nil { if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) { return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", cvg.id, err) @@ -197,6 +209,12 @@ func (cvg *commonVolumeGroup) getConnection(ctx context.Context) (*util.ClusterC return cvg.conn, nil } + if cvg.credentials == nil { + log.DebugLog(ctx, "missing credentials for common volume group %q: %s", cvg, util.CallStack()) + + return nil, errors.New("can not connect to cluster without credentials") + } + conn := &util.ClusterConnection{} err := conn.Connect(cvg.monitors, cvg.credentials) if err != nil { @@ -209,6 +227,29 @@ func (cvg *commonVolumeGroup) getConnection(ctx context.Context) (*util.ClusterC return conn, nil } +func (cvg *commonVolumeGroup) getJournal(ctx context.Context) (journal.VolumeGroupJournal, error) { + if cvg.journal != nil { + return cvg.journal, nil + } + + if cvg.credentials == nil { + log.DebugLog(ctx, "missing credentials for common volume group %q: %s", cvg, util.CallStack()) + + return nil, errors.New("can not connect the journal without credentials") + } + + journalConfig := journal.NewCSIVolumeGroupJournalWithNamespace(cvg.csiDriver, cvg.namespace) + + j, err := journalConfig.Connect(cvg.monitors, cvg.namespace, cvg.credentials) + if err != nil { + return nil, fmt.Errorf("failed to connect to journal: %w", err) + } + + cvg.journal = j + + return j, nil +} + // GetIOContext returns the IOContext for the volume group if it exists, // otherwise it will allocate a new one. // Destroy should be used to free the IOContext. @@ -254,7 +295,12 @@ func (cvg *commonVolumeGroup) Delete(ctx context.Context) error { return fmt.Errorf("failed to get pool for volume group %q: %w", cvg, err) } - err = cvg.journal.UndoReservation(ctx, pool, name, csiID) + j, err := cvg.getJournal(ctx) + if err != nil { + return err + } + + err = j.UndoReservation(ctx, pool, name, csiID) if err != nil /* TODO? !errors.Is(..., err) */ { return fmt.Errorf("failed to undo the reservation for volume group %q: %w", cvg, err) } diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go index da682435652..d268b1142e7 100644 --- a/internal/rbd/group/volume_group.go +++ b/internal/rbd/group/volume_group.go @@ -26,7 +26,6 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/csi-addons/spec/lib/go/volumegroup" - "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/rbd/types" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -36,7 +35,7 @@ var ErrRBDGroupNotConnected = errors.New("RBD group is not connected") // volumeGroup handles all requests for 'rbd group' operations. type volumeGroup struct { - *commonVolumeGroup + commonVolumeGroup // volumes is a list of rbd-images that are part of the group. The ID // of each volume is stored in the journal. @@ -62,12 +61,12 @@ var ( func GetVolumeGroup( ctx context.Context, id string, - j journal.VolumeGroupJournal, + csiDriver string, creds *util.Credentials, volumeResolver types.VolumeResolver, ) (types.VolumeGroup, error) { vg := &volumeGroup{} - err := vg.initCommonVolumeGroup(ctx, id, j, creds) + err := vg.initCommonVolumeGroup(ctx, id, csiDriver, creds) if err != nil { return nil, fmt.Errorf("failed to initialize volume group with id %q: %w", id, err) } @@ -235,7 +234,12 @@ func (vg *volumeGroup) AddVolume(ctx context.Context, vol types.Volume) error { volID: "", } - err = vg.journal.AddVolumesMapping(ctx, pool, csiID.ObjectUUID, toAdd) + j, err := vg.getJournal(ctx) + if err != nil { + return err + } + + err = j.AddVolumesMapping(ctx, pool, csiID.ObjectUUID, toAdd) if err != nil { return fmt.Errorf("failed to add mapping for volume %q to volume group id %q: %w", volID, id, err) } @@ -304,7 +308,12 @@ func (vg *volumeGroup) RemoveVolume(ctx context.Context, vol types.Volume) error toRemove, } - err = vg.journal.RemoveVolumesMapping(ctx, pool, csiID.ObjectUUID, mapping) + j, err := vg.getJournal(ctx) + if err != nil { + return err + } + + err = j.RemoveVolumesMapping(ctx, pool, csiID.ObjectUUID, mapping) if err != nil { return fmt.Errorf("failed to remove mapping for volume %q to volume group id %q: %w", toRemove, id, err) } diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 325943e9e0b..7eaf536c2fd 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -140,22 +140,12 @@ func (mgr *rbdManager) GetVolumeByID(ctx context.Context, id string) (types.Volu } func (mgr *rbdManager) GetVolumeGroupByID(ctx context.Context, id string) (types.VolumeGroup, error) { - vi := &util.CSIIdentifier{} - if err := vi.DecomposeCSIID(id); err != nil { - return nil, fmt.Errorf("failed to parse volume group id %q: %w", id, err) - } - - vgJournal, err := mgr.getVolumeGroupJournal(vi.ClusterID) - if err != nil { - return nil, err - } - creds, err := mgr.getCredentials() if err != nil { return nil, err } - vg, err := rbd_group.GetVolumeGroup(ctx, id, vgJournal, creds, mgr) + vg, err := rbd_group.GetVolumeGroup(ctx, id, mgr.csiID, creds, mgr) if err != nil { return nil, fmt.Errorf("failed to get volume group with id %q: %w", id, err) } @@ -236,7 +226,7 @@ func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (type return nil, fmt.Errorf("failed to generate a unique CSI volume group with uuid for %q: %w", uuid, err) } - vg, err := rbd_group.GetVolumeGroup(ctx, csiID, vgJournal, creds, mgr) + vg, err := rbd_group.GetVolumeGroup(ctx, csiID, mgr.csiID, creds, mgr) if err != nil { return nil, fmt.Errorf("failed to get volume group %q at cluster %q: %w", name, clusterID, err) } From def0862299b88479e4101c590e05d92fdd801ae9 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 12 Sep 2024 15:02:55 +0200 Subject: [PATCH 06/16] rbd: add `rbdVolume.NewSnapshotByID` to clone images by RBD snapshot-id The NewSnapshotByID() function makes it possible to clone a new Snapshot from an existing RBD-image and the ID of an RBD-snapshot on that image. This will be used by the VolumeGroupSnapshot feature, where the ID of an RBD-snapshot is obtained for the RBD-snapshot on the RBD-images. Signed-off-by: Niels de Vos --- internal/rbd/snapshot.go | 134 +++++++++++++++++++++++++++++++++++ internal/rbd/types/volume.go | 5 ++ 2 files changed, 139 insertions(+) diff --git a/internal/rbd/snapshot.go b/internal/rbd/snapshot.go index e903a74f3aa..084bf4d723b 100644 --- a/internal/rbd/snapshot.go +++ b/internal/rbd/snapshot.go @@ -20,9 +20,11 @@ import ( "errors" "fmt" + librbd "github.com/ceph/go-ceph/rbd" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/ceph/ceph-csi/internal/rbd/types" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" ) @@ -176,3 +178,135 @@ func undoSnapshotCloning( return err } + +// NewSnapshotByID creates a new rbdSnapshot from the rbdVolume. +// +// Parameters: +// - name of the new rbd-image backing the snapshot +// - id of the rbd-snapshot to clone +// +// FIXME: When resolving the Snapshot, the RbdImageName will be set to the name +// of the parent image. This is can cause issues when not accounting for that +// and the Snapshot is deleted; instead of deleting the snapshot image, the +// parent image is removed. +// +//nolint:gocyclo,cyclop // TODO: reduce complexity. +func (rv *rbdVolume) NewSnapshotByID( + ctx context.Context, + cr *util.Credentials, + name string, + id uint64, +) (types.Snapshot, error) { + snap := rv.toSnapshot() + snap.RequestName = name + + srcVolID, err := rv.GetID(ctx) + if err != nil { + return nil, err + } + + snap.SourceVolumeID = srcVolID + + // reserveSnap sets snap.{RbdSnapName,ReservedID,VolID} + err = reserveSnap(ctx, snap, rv, cr) + if err != nil { + return nil, fmt.Errorf("failed to create a reservation in the journal for snapshot image %q: %w", snap, err) + } + defer func() { + if err != nil { + undoErr := undoSnapReservation(ctx, snap, cr) + if undoErr != nil { + log.WarningLog(ctx, "failed undoing reservation of snapshot %q: %v", name, undoErr) + } + } + }() + + // a new snapshot image will be created, needs to have a unique name + snap.RbdImageName = snap.RbdSnapName + + err = rv.Connect(cr) + if err != nil { + return nil, err + } + + err = rv.openIoctx() + if err != nil { + return nil, err + } + + options, err := rv.constructImageOptions(ctx) + if err != nil { + return nil, err + } + defer options.Destroy() + + err = options.SetUint64(librbd.ImageOptionCloneFormat, 2) + if err != nil { + return nil, err + } + + // indicator to remove the snapshot after a failure + removeSnap := true + var snapImage *librbd.Snapshot + + log.DebugLog(ctx, "going to clone snapshot image %q from image %q with snapshot ID %d", snap, rv, id) + + err = librbd.CloneImageByID(rv.ioctx, rv.RbdImageName, id, rv.ioctx, snap.RbdImageName, options) + if err != nil && !errors.Is(librbd.ErrExist, err) { + log.ErrorLog(ctx, "failed to clone snapshot %q with id %d: %v", snap, id, err) + + return nil, fmt.Errorf("failed to clone %q with snapshot id %d as new image %q: %w", rv.RbdImageName, id, snap, err) + } + defer func() { + if !removeSnap { + // success, no need to remove the snapshot image + return + } + + if snapImage != nil { + err = snapImage.Remove() + if err != nil { + log.ErrorLog(ctx, "failed to remove snapshot of image %q after failure: %v", snap, err) + } + } + + err = librbd.RemoveImage(rv.ioctx, snap.RbdImageName) + if err != nil { + log.ErrorLog(ctx, "failed to remove snapshot image %q after failure: %v", snap, err) + } + }() + + // update the snapshot image in the journal, after the image info is updated + j, err := snapJournal.Connect(snap.Monitors, snap.RadosNamespace, cr) + if err != nil { + return nil, fmt.Errorf("snapshot image %q failed to connect to journal: %w", snap, err) + } + defer j.Destroy() + + err = snap.Connect(cr) + if err != nil { + return nil, fmt.Errorf("failed to connect snapshot image %q: %w", snap, err) + } + defer snap.Destroy(ctx) + + image, err := snap.open() + if err != nil { + return nil, fmt.Errorf("failed to open snapshot image %q: %w", snap, err) + } + defer image.Close() + + snapImage, err = image.CreateSnapshot(snap.RbdSnapName) + if err != nil && !errors.Is(librbd.ErrExist, err) { + return nil, fmt.Errorf("failed to create snapshot on image %q: %w", snap, err) + } + + err = snap.repairImageID(ctx, j, true) + if err != nil { + return nil, fmt.Errorf("failed to repair image id for snapshot image %q: %w", snap, err) + } + + // all ok, don't remove the snapshot image in a defer statement + removeSnap = false + + return snap, nil +} diff --git a/internal/rbd/types/volume.go b/internal/rbd/types/volume.go index fb2b6627546..1f235bf16ba 100644 --- a/internal/rbd/types/volume.go +++ b/internal/rbd/types/volume.go @@ -21,6 +21,8 @@ import ( "time" "github.com/container-storage-interface/spec/lib/go/csi" + + "github.com/ceph/ceph-csi/internal/util" ) //nolint:interfacebloat // more than 10 methods are needed for the interface @@ -59,4 +61,7 @@ type Volume interface { // ToMirror converts the Volume to a Mirror. ToMirror() (Mirror, error) + + // NewSnapshotByID creates a new Snapshot object based on the details of the Volume. + NewSnapshotByID(ctx context.Context, cr *util.Credentials, name string, id uint64) (Snapshot, error) } From 7f18e7a2b2b76ba9edd4a9ec936691687be7470f Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 4 Oct 2024 11:54:11 +0200 Subject: [PATCH 07/16] rbd: add VolumeGroup.CreateSnapshots() implementation When the rbd.Manager creates a VolumeGroupSnapshot, each RBD-snapshot that is created as part of the RBD-group needs to be cloned into its own RBD-image that will be used as a CSI Snapshot. The VolumeGroup.CreateSnapshots() creates the RBD-group snapshot and returns a list of the Snapshot structs. Signed-off-by: Niels de Vos --- internal/rbd/group/volume_group.go | 91 +++++++++++++++++++++++++++++- internal/rbd/types/group.go | 7 +++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go index d268b1142e7..cdb6a8cf0bf 100644 --- a/internal/rbd/group/volume_group.go +++ b/internal/rbd/group/volume_group.go @@ -241,7 +241,8 @@ func (vg *volumeGroup) AddVolume(ctx context.Context, vol types.Volume) error { err = j.AddVolumesMapping(ctx, pool, csiID.ObjectUUID, toAdd) if err != nil { - return fmt.Errorf("failed to add mapping for volume %q to volume group id %q: %w", volID, id, err) + return fmt.Errorf("failed to add mapping for volume %q to volume group id %q: %w", + volID, id, err) } return nil @@ -315,7 +316,8 @@ func (vg *volumeGroup) RemoveVolume(ctx context.Context, vol types.Volume) error err = j.RemoveVolumesMapping(ctx, pool, csiID.ObjectUUID, mapping) if err != nil { - return fmt.Errorf("failed to remove mapping for volume %q to volume group id %q: %w", toRemove, id, err) + return fmt.Errorf("failed to remove mapping for volume %q to volume group id %q: %w", + toRemove, id, err) } return nil @@ -324,3 +326,88 @@ func (vg *volumeGroup) RemoveVolume(ctx context.Context, vol types.Volume) error func (vg *volumeGroup) ListVolumes(ctx context.Context) ([]types.Volume, error) { return vg.volumes, nil } + +// CreateSnapshots makes consistent snapshots of all the volumes in the volume group. +func (vg *volumeGroup) CreateSnapshots( + ctx context.Context, + cr *util.Credentials, + name string, +) ([]types.Snapshot, error) { + group, err := vg.GetName(ctx) + if err != nil { + return nil, err + } + + ioctx, err := vg.GetIOContext(ctx) + if err != nil { + return nil, err + } + + err = librbd.GroupSnapCreate(ioctx, group, name) + if err != nil { + if !errors.Is(err, librbd.ErrExist) { + return nil, fmt.Errorf("failed to create volume group snapshot %q: %w", name, err) + } + + log.DebugLog(ctx, "ignoring error while creating volume group snapshot %q: %v", vg, err) + } + defer func() { + // always remove the groups-snapshot on function exit, it is not used anymore afterwards + cleanupErr := librbd.GroupSnapRemove(ioctx, group, name) + if cleanupErr != nil { + log.ErrorLog(ctx, "failed to remove temporary volume group snapshot %q: %v", + name, cleanupErr) + } + }() + + info, err := librbd.GroupSnapGetInfo(ioctx, group, name) + if err != nil { + return nil, fmt.Errorf("failed to get info for volume group snapshot %q: %w", + vg.String()+"@"+name, err) + } + + snapshots := make([]types.Snapshot, len(info.Snapshots)) + defer func() { + // free all created snapshot objects in case of a failure + if err == nil { + return + } + + for _, snapshot := range snapshots { + snapshot.Destroy(ctx) + } + }() + + // Loop though all the RBD-snapshots in the group, and find the volume + // that was used to create the snapshot. Once found, use the volume to + // create a new RBD-image from the RBD-snapshot. + for i, snap := range info.Snapshots { + for _, volume := range vg.volumes { + var volName string + + volName, err = volume.GetName(ctx) + if err != nil { + return nil, fmt.Errorf( + "failed to get name for volume %q: %w", volume, err) + } + if volName != snap.Name { + // the volume isn't the snapshot-source, continue with the next one + continue + } + + // yay, volume for the RBD-snapshot found! + snapName := fmt.Sprintf("%s-snap-%d", group, i) + snapshots[i], err = volume.NewSnapshotByID(ctx, cr, snapName, snap.SnapID) + if err != nil { + return nil, fmt.Errorf( + "failed to create snapshot for image %q with snapshot id %d: %w", + snap.Name, snap.SnapID, err) + } + + // done, no need to try more volumes in the loop + break + } + } + + return snapshots, nil +} diff --git a/internal/rbd/types/group.go b/internal/rbd/types/group.go index 36f89e807f5..08183f9fb63 100644 --- a/internal/rbd/types/group.go +++ b/internal/rbd/types/group.go @@ -21,6 +21,8 @@ import ( "github.com/ceph/go-ceph/rados" "github.com/csi-addons/spec/lib/go/volumegroup" + + "github.com/ceph/ceph-csi/internal/util" ) type journalledObject interface { @@ -66,4 +68,9 @@ type VolumeGroup interface { // ListVolumes returns a slice with all Volumes in the VolumeGroup. ListVolumes(ctx context.Context) ([]Volume, error) + + // CreateSnapshots creates Snapshots of all Volume in the VolumeGroup. + // The Snapshots are crash consistent, and created as a consistency + // group. + CreateSnapshots(ctx context.Context, cr *util.Credentials, name string) ([]Snapshot, error) } From c91a8354f2a957b1751cbdae7a5605147a10dd5e Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 21 Aug 2024 15:36:08 +0200 Subject: [PATCH 08/16] rbd: add VolumeGroupSnapshot type The VolumeGroupSnapshot type will be used by the rbd.Manager to create, inspect and delete VolumeGroupSnapshos. Signed-off-by: Niels de Vos --- internal/rbd/group/group_snapshot.go | 238 ++++++++++++++++++++ internal/rbd/types/volume_group_snapshot.go | 48 ++++ 2 files changed, 286 insertions(+) create mode 100644 internal/rbd/group/group_snapshot.go create mode 100644 internal/rbd/types/volume_group_snapshot.go diff --git a/internal/rbd/group/group_snapshot.go b/internal/rbd/group/group_snapshot.go new file mode 100644 index 00000000000..7abb237addd --- /dev/null +++ b/internal/rbd/group/group_snapshot.go @@ -0,0 +1,238 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package group + +import ( + "context" + "fmt" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/ceph/ceph-csi/internal/rbd/types" + "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" +) + +// volumeGroupSnapshot handles all requests for 'rbd group snap' operations. +type volumeGroupSnapshot struct { + commonVolumeGroup + + // snapshots is a list of rbd-images that are part of the group. The ID + // of each snapshot is stored in the journal. + snapshots []types.Snapshot + + // snapshotsToFree contains Snapshots that were resolved during + // GetVolumeGroupSnapshot. + snapshotsToFree []types.Snapshot +} + +// verify that volumeGroupSnapshot implements the VolumeGroupSnapshot interface. +var _ types.VolumeGroupSnapshot = &volumeGroupSnapshot{} + +// GetVolumeGroupSnapshot initializes a new VolumeGroupSnapshot object that can +// be used to inspect and delete a group of snapshots that was created by a +// VolumeGroup. +func GetVolumeGroupSnapshot( + ctx context.Context, + id string, + csiDriver string, + creds *util.Credentials, + snapshotResolver types.SnapshotResolver, +) (types.VolumeGroupSnapshot, error) { + cleanVGS := true + + vgs := &volumeGroupSnapshot{} + err := vgs.initCommonVolumeGroup(ctx, id, csiDriver, creds) + if err != nil { + return nil, fmt.Errorf("failed to initialize volume group snapshot with id %q: %w", id, err) + } + defer func() { + if cleanVGS { + vgs.Destroy(ctx) + } + }() + + attrs, err := vgs.getVolumeGroupAttributes(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vgs, err) + } + + var snapshots []types.Snapshot + // it is needed to free the previously allocated snapshots in case of an error + defer func() { + // snapshotsToFree is empty in case of an error, let .Destroy() handle it otherwise + if len(vgs.snapshotsToFree) > 0 { + return + } + + for _, s := range snapshots { + s.Destroy(ctx) + } + }() + for snapID := range attrs.VolumeMap { + snap, err := snapshotResolver.GetSnapshotByID(ctx, snapID) + if err != nil { + // free the previously allocated snapshots + for _, s := range snapshots { + s.Destroy(ctx) + } + + return nil, fmt.Errorf("failed to resolve snapshot image %q for volume group snapshot %q: %w", snapID, vgs, err) + } + + log.DebugLog(ctx, "resolved snapshot id %q to snapshot %q", snapID, snap) + + snapshots = append(snapshots, snap) + } + + vgs.snapshots = snapshots + // all allocated snapshots need to be free'd at Destroy() time + vgs.snapshotsToFree = snapshots + + cleanVGS = false + log.DebugLog(ctx, "GetVolumeGroupSnapshot(%s) returns %+v", id, *vgs) + + return vgs, nil +} + +// NewVolumeGroupSnapshot creates a new VolumeGroupSnapshot object with the +// given slice of Snapshots and adds the objectmapping to the journal. +func NewVolumeGroupSnapshot( + ctx context.Context, + id string, + csiDriver string, + creds *util.Credentials, + snapshots []types.Snapshot, +) (types.VolumeGroupSnapshot, error) { + cleanupVGS := true + + vgs := &volumeGroupSnapshot{} + err := vgs.initCommonVolumeGroup(ctx, id, csiDriver, creds) + if err != nil { + return nil, fmt.Errorf("failed to initialize volume group snapshot with id %q: %w", id, err) + } + defer func() { + if cleanupVGS { + vgs.Destroy(ctx) + } + }() + + vgs.snapshots = snapshots + vgs.snapshotsToFree = snapshots + + _ /* attrs */, err = vgs.getVolumeGroupAttributes(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vgs, err) + } + + volumeMap := make(map[string]string, len(snapshots)) + + // add the CSI handles of each snapshot to the journal + for _, snapshot := range snapshots { + handle, snapErr := snapshot.GetID(ctx) + if snapErr != nil { + return nil, fmt.Errorf("failed to get ID for snapshot %q of volume group snapshot %q: %w", snapshot, vgs, snapErr) + } + + name, snapErr := snapshot.GetName(ctx) + if snapErr != nil { + return nil, fmt.Errorf("failed to get name for snapshot %q of volume group snapshot %q: %w", snapshot, vgs, snapErr) + } + + volumeMap[handle] = name + } + + j, err := vgs.getJournal(ctx) + if err != nil { + return nil, err + } + + err = j.AddVolumesMapping(ctx, vgs.pool, vgs.objectUUID, volumeMap) + if err != nil { + return nil, fmt.Errorf("failed to add volume mapping for volume group snapshot %q: %w", vgs, err) + } + + // all done successfully, no need to cleanup the returned vgs + cleanupVGS = false + + return vgs, nil +} + +// ToCSI creates a CSI type for the VolumeGroupSnapshot. +func (vgs *volumeGroupSnapshot) ToCSI(ctx context.Context) (*csi.VolumeGroupSnapshot, error) { + snapshots, err := vgs.ListSnapshots(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list snapshots for volume group %q: %w", vgs, err) + } + + csiSnapshots := make([]*csi.Snapshot, len(snapshots)) + for i, snap := range snapshots { + csiSnapshots[i], err = snap.ToCSI(ctx) + if err != nil { + return nil, fmt.Errorf("failed to convert snapshot %q to CSI type: %w", snap, err) + } + } + + id, err := vgs.GetID(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get id for volume group snapshot %q: %w", vgs, err) + } + + ct, err := vgs.GetCreationTime(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get creation time for volume group snapshot %q: %w", vgs, err) + } + + return &csi.VolumeGroupSnapshot{ + GroupSnapshotId: id, + Snapshots: csiSnapshots, + CreationTime: timestamppb.New(*ct), + ReadyToUse: true, + }, nil +} + +// Destroy frees the resources used by the volumeGroupSnapshot. +func (vgs *volumeGroupSnapshot) Destroy(ctx context.Context) { + // free the volumes that were allocated in GetVolumeGroup() + if len(vgs.snapshotsToFree) > 0 { + for _, volume := range vgs.snapshotsToFree { + volume.Destroy(ctx) + } + vgs.snapshotsToFree = make([]types.Snapshot, 0) + } + + vgs.commonVolumeGroup.Destroy(ctx) +} + +// Delete removes all snapshots and eventually the volume group snapshot. +func (vgs *volumeGroupSnapshot) Delete(ctx context.Context) error { + for _, snapshot := range vgs.snapshots { + log.DebugLog(ctx, "deleting snapshot image %q for volume group snapshot %q", snapshot, vgs) + + err := snapshot.Delete(ctx) + if err != nil { + return fmt.Errorf("failed to delete snapshot %q as part of volume groups snapshot %q: %w", snapshot, vgs, err) + } + } + + return vgs.commonVolumeGroup.Delete(ctx) +} + +func (vgs *volumeGroupSnapshot) ListSnapshots(ctx context.Context) ([]types.Snapshot, error) { + return vgs.snapshots, nil +} diff --git a/internal/rbd/types/volume_group_snapshot.go b/internal/rbd/types/volume_group_snapshot.go new file mode 100644 index 00000000000..7d9d42e2870 --- /dev/null +++ b/internal/rbd/types/volume_group_snapshot.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "context" + "time" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// VolumeGroupSnapshot provide functions to inspect and modify a +// VolumeGroupSnapshot. The rbd.Manager can be used to create or otherwise +// obtain a VolumeGroupSnapshot struct. +type VolumeGroupSnapshot interface { + journalledObject + + // Destroy frees the resources used by the VolumeGroupSnapshot. + Destroy(ctx context.Context) + + // Delete removes the VolumeGroupSnapshot from the storage backend. + Delete(ctx context.Context) error + + // ToCSI returns the VolumeGroupSnapshot struct in CSI format. + ToCSI(ctx context.Context) (*csi.VolumeGroupSnapshot, error) + + // GetCreationTime retrurns the time when the VolumeGroupSnapshot was + // created. + GetCreationTime(ctx context.Context) (*time.Time, error) + + // ListSnapshots returns a slice with all Snapshots in the + // VolumeGroupSnapshot. + ListSnapshots(ctx context.Context) ([]Snapshot, error) +} From 1240204225e46c8b5fae8c5f79096a72704ef04f Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 4 Oct 2024 15:40:33 +0200 Subject: [PATCH 09/16] rbd: add manager GetSnapshotByID and SnapshotResolver interface A (CSI) VolumeGroupSnapshot object contains references to Snapshot IDs (or CSI Snapshot handles). In order to work with a VolumeGroupSnapshot struct, the Snapshot IDs need to be resolved into rbdSnapshot structs. Signed-off-by: Niels de Vos --- internal/rbd/manager.go | 25 +++++++++++++++++++++++++ internal/rbd/types/manager.go | 9 +++++++++ 2 files changed, 34 insertions(+) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 7eaf536c2fd..1830df79b25 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -139,6 +139,31 @@ func (mgr *rbdManager) GetVolumeByID(ctx context.Context, id string) (types.Volu return volume, nil } +func (mgr *rbdManager) GetSnapshotByID(ctx context.Context, id string) (types.Snapshot, error) { + creds, err := mgr.getCredentials() + if err != nil { + return nil, err + } + + snapshot, err := genSnapFromSnapID(ctx, id, creds, mgr.secrets) + if err != nil { + switch { + case errors.Is(err, ErrImageNotFound): + err = fmt.Errorf("volume %s not found: %w", id, err) + + return nil, err + case errors.Is(err, util.ErrPoolNotFound): + err = fmt.Errorf("pool %s not found for %s: %w", snapshot.Pool, id, err) + + return nil, err + default: + return nil, fmt.Errorf("failed to get volume from id %q: %w", id, err) + } + } + + return snapshot, nil +} + func (mgr *rbdManager) GetVolumeGroupByID(ctx context.Context, id string) (types.VolumeGroup, error) { creds, err := mgr.getCredentials() if err != nil { diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go index 2e9521570b5..2d9eb36e3be 100644 --- a/internal/rbd/types/manager.go +++ b/internal/rbd/types/manager.go @@ -26,6 +26,12 @@ type VolumeResolver interface { GetVolumeByID(ctx context.Context, id string) (Volume, error) } +// SnapshotResolver can be used to construct a Snapshot from a CSI SnapshotId. +type SnapshotResolver interface { + // GetSnapshotByID uses the CSI SnapshotId to resolve the returned Snapshot. + GetSnapshotByID(ctx context.Context, id string) (Snapshot, error) +} + // Manager provides a way for other packages to get Volumes and VolumeGroups. // It handles the operations on the backend, and makes sure the journal // reflects the expected state. @@ -33,6 +39,9 @@ type Manager interface { // VolumeResolver is fully implemented by the Manager. VolumeResolver + // SnapshotResolver is fully implemented by the Manager. + SnapshotResolver + // Destroy frees all resources that the Manager allocated. Destroy(ctx context.Context) From e415b4bf82db57f22bccfe7d0480115b27b722e3 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 28 Aug 2024 17:46:22 +0200 Subject: [PATCH 10/16] rbd: add manager.CreateVolumeGroupSnapshot() Implement the CreateVolumeGroupSnapshot for the rbd.Manager. A Group Controller Server can use the rbd.Manager to create VolumeGroupSnapshots in an easy an idempotent way. Signed-off-by: Niels de Vos --- internal/rbd/manager.go | 156 ++++++++++++++++++++++++++++++++++ internal/rbd/types/manager.go | 7 ++ 2 files changed, 163 insertions(+) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 1830df79b25..ebb20b6f5b8 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -114,6 +114,57 @@ func (mgr *rbdManager) getVolumeGroupJournal(clusterID string) (journal.VolumeGr return vgJournal, nil } +// getGroupUUID checks if a UUID in the volume group journal is already +// reserved. If none is reserved, a new reservation is made. Upon exit of +// getGroupUUID, the function returns: +// 1. the UUID that was reserved +// 2. an undo() function that reverts the reservation (if that succeeded), should be called in a defer +// 3. an error or nil. +func (mgr *rbdManager) getGroupUUID( + ctx context.Context, + clusterID, journalPool, name, prefix string, +) (string, func(), error) { + nothingToUndo := func() { + // the reservation was not done, no need to undo the reservation + } + + vgJournal, err := mgr.getVolumeGroupJournal(clusterID) + if err != nil { + return "", nothingToUndo, err + } + + vgsData, err := vgJournal.CheckReservation(ctx, journalPool, name, prefix) + if err != nil { + return "", nothingToUndo, fmt.Errorf("failed to check reservation for group %q: %w", name, err) + } + + var uuid string + if vgsData != nil && vgsData.GroupUUID != "" { + uuid = vgsData.GroupUUID + } else { + log.DebugLog(ctx, "the journal does not contain a reservation for group %q yet", name) + + uuid, _ /*vgsName*/, err = vgJournal.ReserveName(ctx, journalPool, name, prefix) + if err != nil { + return "", nothingToUndo, fmt.Errorf("failed to reserve a UUID for group %q: %w", name, err) + } + } + + log.DebugLog(ctx, "got UUID %q for group %q", uuid, name) + + // undo contains the cleanup that should be done by the caller when the + // reservation was made, and further actions fulfilling the final + // request failed + undo := func() { + err = vgJournal.UndoReservation(ctx, journalPool, uuid, name) + if err != nil { + log.ErrorLog(ctx, "failed to undo the reservation for group %q: %w", name, err) + } + } + + return uuid, undo, nil +} + func (mgr *rbdManager) GetVolumeByID(ctx context.Context, id string) (types.Volume, error) { creds, err := mgr.getCredentials() if err != nil { @@ -268,3 +319,108 @@ func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (type return vg, nil } + +func (mgr *rbdManager) CreateVolumeGroupSnapshot( + ctx context.Context, + vg types.VolumeGroup, + name string, +) (types.VolumeGroupSnapshot, error) { + pool, err := vg.GetPool(ctx) + if err != nil { + return nil, err + } + + // groupNamePrefix is an optional parameter, can be an empty string + prefix := mgr.parameters["groupNamePrefix"] + + clusterID, err := vg.GetClusterID(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get cluster id for volume group snapshot %q: %w", vg, err) + } + + uuid, freeUUID, err := mgr.getGroupUUID(ctx, clusterID, pool, name, prefix) + if err != nil { + return nil, fmt.Errorf("failed to get a UUID for volume group snapshot %q: %w", vg, err) + } + defer func() { + // no error, no need to undo the reservation + if err == nil { + return + } + + freeUUID() + }() + + monitors, err := util.Mons(util.CsiConfigFile, clusterID) + if err != nil { + return nil, fmt.Errorf("failed to find MONs for cluster %q: %w", clusterID, err) + } + + _ /*journalPoolID*/, poolID, err := util.GetPoolIDs(ctx, monitors, pool, pool, mgr.creds) + if err != nil { + return nil, fmt.Errorf("failed to get PoolID for %q: %w", pool, err) + } + + groupID, err := util.GenerateVolID(ctx, monitors, mgr.creds, poolID, pool, clusterID, uuid) + if err != nil { + return nil, fmt.Errorf("failed to generate a unique CSI volume group with uuid for %q: %w", uuid, err) + } + + vgs, err := rbd_group.GetVolumeGroupSnapshot(ctx, groupID, mgr.csiID, mgr.creds, mgr) + if vgs != nil { + log.DebugLog(ctx, "found existing volume group snapshot %q for id %q", vgs, groupID) + + // validate the contents of the vgs + snapshots, vgsErr := vgs.ListSnapshots(ctx) + if vgsErr != nil { + return nil, fmt.Errorf("failed to list snapshots of existing volume group snapshot %q: %w", vgs, vgsErr) + } + + volumes, vgErr := vg.ListVolumes(ctx) + if vgErr != nil { + return nil, fmt.Errorf("failed to list volumes of volume group %q: %w", vg, vgErr) + } + + // return the existing vgs if the contents matches + // TODO: improve contents verification, length is a very minimal check + if len(snapshots) == len(volumes) { + log.DebugLog(ctx, "existing volume group snapshot %q contains %d snapshots", vgs, len(snapshots)) + + return vgs, nil + } + } else if err != nil && !errors.Is(ErrImageNotFound, err) { + // ErrImageNotFound can be returned if the VolumeGroupSnapshot + // could not be found. It is expected that it does not exist + // yet, in which case it will be created below. + return nil, fmt.Errorf("failed to check for existing volume group snapshot with id %q: %w", groupID, err) + } + + snapshots, err := vg.CreateSnapshots(ctx, mgr.creds, groupID) + if err != nil { + return nil, fmt.Errorf("failed to create volume group snapshot %q: %w", name, err) + } + defer func() { + // cleanup created snapshots in case there was a failure + if err == nil { + return + } + + for _, snap := range snapshots { + delErr := snap.Delete(ctx) + if delErr != nil { + log.ErrorLog(ctx, "failed to delete snapshot %q: %v", snap, delErr) + } + } + }() + + log.DebugLog(ctx, "volume group snapshot %q contains %d snapshots: %v", name, len(snapshots), snapshots) + + vgs, err = rbd_group.NewVolumeGroupSnapshot(ctx, groupID, mgr.csiID, mgr.creds, snapshots) + if err != nil { + return nil, fmt.Errorf("failed to create new volume group snapshot %q: %w", name, err) + } + + log.DebugLog(ctx, "volume group snapshot %q has been created", vgs) + + return vgs, nil +} diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go index 2d9eb36e3be..c2541b493d6 100644 --- a/internal/rbd/types/manager.go +++ b/internal/rbd/types/manager.go @@ -52,4 +52,11 @@ type Manager interface { // CreateVolumeGroup allocates a new VolumeGroup in the backend storage // and records details about it in the journal. CreateVolumeGroup(ctx context.Context, name string) (VolumeGroup, error) + + // CreateVolumeGroupSnapshot instructs the Manager to create a + // VolumeGroupSnapshot from the VolumeGroup. All snapshots in the + // returned VolumeGroupSnapshot have been taken while I/O on the + // VolumeGroup was paused, the snapshots in the group are crash + // consistent. + CreateVolumeGroupSnapshot(ctx context.Context, vg VolumeGroup, name string) (VolumeGroupSnapshot, error) } From e058ca2d6ce4226bd5fe7a0758cbe0a04a3077b4 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 17 Oct 2024 17:31:45 +0200 Subject: [PATCH 11/16] rbd: implement Manager.GetVolumeGroupSnapshotByID The GetVolumeGroupSnapshotByID function makes it possible to get a VolumeGroupSnapshot object from the Manager by passing a request-id. This makes it simple for the Group Controller Server to check if a VolumeGroupSnapshot already exists, so it is not needed to try and re-create an existing one. Signed-off-by: Niels de Vos --- internal/rbd/manager.go | 17 +++++++++++++++++ internal/rbd/types/manager.go | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index ebb20b6f5b8..d637c89a796 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -320,6 +320,23 @@ func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (type return vg, nil } +func (mgr *rbdManager) GetVolumeGroupSnapshotByID( + ctx context.Context, + id string, +) (types.VolumeGroupSnapshot, error) { + creds, err := mgr.getCredentials() + if err != nil { + return nil, err + } + + vgs, err := rbd_group.GetVolumeGroupSnapshot(ctx, id, mgr.csiID, creds, mgr) + if err != nil { + return nil, fmt.Errorf("failed to get volume group with id %q: %w", id, err) + } + + return vgs, nil +} + func (mgr *rbdManager) CreateVolumeGroupSnapshot( ctx context.Context, vg types.VolumeGroup, diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go index c2541b493d6..105e2837431 100644 --- a/internal/rbd/types/manager.go +++ b/internal/rbd/types/manager.go @@ -53,6 +53,10 @@ type Manager interface { // and records details about it in the journal. CreateVolumeGroup(ctx context.Context, name string) (VolumeGroup, error) + // GetVolumeGroupSnapshotByID resolves the VolumeGroupSnapshot from the + // CSI id/handle. + GetVolumeGroupSnapshotByID(ctx context.Context, id string) (VolumeGroupSnapshot, error) + // CreateVolumeGroupSnapshot instructs the Manager to create a // VolumeGroupSnapshot from the VolumeGroup. All snapshots in the // returned VolumeGroupSnapshot have been taken while I/O on the From 54061e58ad5341426c78ab5effae1423999ee11a Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 17 Oct 2024 17:34:24 +0200 Subject: [PATCH 12/16] rbd: add Manager.GetVolumeGroupSnapshotByName The Group Controller Server may need to fetch a VolumeGroupSnapshot that was statically provisioned. In that case, only the name of the VolumeGroupSnapshot is known and should be resolved to an object. Signed-off-by: Niels de Vos --- internal/rbd/manager.go | 62 +++++++++++++++++++++++++++++++++++ internal/rbd/types/manager.go | 4 +++ 2 files changed, 66 insertions(+) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index d637c89a796..8b0d6a06448 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -337,6 +337,68 @@ func (mgr *rbdManager) GetVolumeGroupSnapshotByID( return vgs, nil } +func (mgr *rbdManager) GetVolumeGroupSnapshotByName( + ctx context.Context, + name string, +) (types.VolumeGroupSnapshot, error) { + pool, ok := mgr.parameters["pool"] + if !ok || pool == "" { + return nil, errors.New("required 'pool' option missing in volume group parameters") + } + + // groupNamePrefix is an optional parameter, can be an empty string + prefix := mgr.parameters["groupNamePrefix"] + + clusterID, err := util.GetClusterID(mgr.parameters) + if err != nil { + return nil, fmt.Errorf("failed to get cluster-id: %w", err) + } + + uuid, freeUUID, err := mgr.getGroupUUID(ctx, clusterID, pool, name, prefix) + if err != nil { + return nil, fmt.Errorf("failed to get a UUID for volume group snapshot %q: %w", name, err) + } + defer func() { + // no error, no need to undo the reservation + if err == nil { + return + } + + freeUUID() + }() + + monitors, err := util.Mons(util.CsiConfigFile, clusterID) + if err != nil { + return nil, fmt.Errorf("failed to find MONs for cluster %q: %w", clusterID, err) + } + + _ /*journalPoolID*/, poolID, err := util.GetPoolIDs(ctx, monitors, pool, pool, mgr.creds) + if err != nil { + return nil, fmt.Errorf("failed to get the pool for volume group snapshot with uuid for %q: %w", uuid, err) + } + + csiID, err := util.GenerateVolID(ctx, monitors, mgr.creds, poolID, pool, clusterID, uuid) + if err != nil { + return nil, fmt.Errorf("failed to generate a unique CSI volume group with uuid %q: %w", uuid, err) + } + + vgs, err := rbd_group.GetVolumeGroupSnapshot(ctx, csiID, mgr.csiID, mgr.creds, mgr) + if err != nil { + return nil, fmt.Errorf("failed to get existing volume group snapshot with uuid %q: %w", uuid, err) + } + + snapshots, err := vgs.ListSnapshots(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get snapshots for volume group snapshot %q: %w", vgs, err) + } + + if len(snapshots) == 0 { + return nil, fmt.Errorf("volume group snapshot %q is incomplete, it has no snapshots", vgs) + } + + return vgs, nil +} + func (mgr *rbdManager) CreateVolumeGroupSnapshot( ctx context.Context, vg types.VolumeGroup, diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go index 105e2837431..273bd58109f 100644 --- a/internal/rbd/types/manager.go +++ b/internal/rbd/types/manager.go @@ -57,6 +57,10 @@ type Manager interface { // CSI id/handle. GetVolumeGroupSnapshotByID(ctx context.Context, id string) (VolumeGroupSnapshot, error) + // GetVolumeGroupSnapshotByName resolves the VolumeGroupSnapshot by the + // name (like the request-id). + GetVolumeGroupSnapshotByName(ctx context.Context, name string) (VolumeGroupSnapshot, error) + // CreateVolumeGroupSnapshot instructs the Manager to create a // VolumeGroupSnapshot from the VolumeGroup. All snapshots in the // returned VolumeGroupSnapshot have been taken while I/O on the From 14ab5db14d44c1e1925666f4abe6b0c8f0333799 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 1 Oct 2024 19:06:50 +0200 Subject: [PATCH 13/16] rbd: fix snapshot deletion by resolving image names correctly When creating a Snapshot with the new NewSnapshotByID() function, the name of the RBD-image that is created is the same as the name of the Snapshot. The `RbdImageName` points to the name of parent image, which causes deleting the Snapshot to delete the parent image instead. Correcting the `RbdImageName` and setting it to the `RbdSnapName` makes sure that upon deletion, the Snapshot RBD-image is removed, and not the parent image. Signed-off-by: Niels de Vos --- internal/rbd/manager.go | 5 +++++ internal/rbd/snapshot.go | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 8b0d6a06448..73f783fb027 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -212,6 +212,11 @@ func (mgr *rbdManager) GetSnapshotByID(ctx context.Context, id string) (types.Sn } } + // FIXME: The snapshot will have RbdImageName set to the image that was + // used as source. This is not correct for group snapshots images, and + // need to be fixed. See rbdVolume.NewSnapshotByID() for more details. + snapshot.RbdImageName = snapshot.RbdSnapName + return snapshot, nil } diff --git a/internal/rbd/snapshot.go b/internal/rbd/snapshot.go index 084bf4d723b..209299e5c50 100644 --- a/internal/rbd/snapshot.go +++ b/internal/rbd/snapshot.go @@ -221,7 +221,12 @@ func (rv *rbdVolume) NewSnapshotByID( } }() - // a new snapshot image will be created, needs to have a unique name + // A new snapshot image will be created, and needs to have a unique + // name. + // FIXME: the journal contains rv.RbdImageName as SourceName. When + // resolving the snapshot image, snap.RbdImageName will be set to the + // original RbdImageName/SourceName (incorrect). This is fixed-up in + // rbdManager.GetSnapshotByID(), this needs to be done cleaner. snap.RbdImageName = snap.RbdSnapName err = rv.Connect(cr) From 1da6527ba75bcbc534c9291ac0fde105a3684a21 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 4 Oct 2024 12:21:57 +0200 Subject: [PATCH 14/16] rbd: implement CSI Group Controller Server Signed-off-by: Niels de Vos --- internal/rbd/group_controllerserver.go | 218 +++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 internal/rbd/group_controllerserver.go diff --git a/internal/rbd/group_controllerserver.go b/internal/rbd/group_controllerserver.go new file mode 100644 index 00000000000..6107b606251 --- /dev/null +++ b/internal/rbd/group_controllerserver.go @@ -0,0 +1,218 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ceph/ceph-csi/internal/rbd/types" + "github.com/ceph/ceph-csi/internal/util/log" +) + +// CreateVolumeGroupSnapshot receives a list of volume handles and is requested +// to create a list of snapshots that are created at the same time. This is +// similar (although not exactly the same) to consistency groups. +// +// RBD has a limitation that an image can only belong to a single group. It is +// therefore required to create a temporary group, add all images, create the +// group snapshot and remove all images from the group again. This leaves the +// group and its snapshot around, the group snapshot can be inspected to list +// the snapshots of the images. +func (cs *ControllerServer) CreateVolumeGroupSnapshot( + ctx context.Context, + req *csi.CreateVolumeGroupSnapshotRequest, +) (*csi.CreateVolumeGroupSnapshotResponse, error) { + var ( + err error + vg types.VolumeGroup + groupSnapshot types.VolumeGroupSnapshot + + // the VG and VGS should not have the same name + vgName = req.GetName() + "-vg" // stable temporary name + vgsName = req.GetName() + ) + + mgr := NewManager(cs.Driver.GetInstanceID(), req.GetParameters(), req.GetSecrets()) + defer mgr.Destroy(ctx) + + // resolve all volumes, free them later on + volumes := make([]types.Volume, len(req.GetSourceVolumeIds())) + defer func() { + for _, volume := range volumes { + if vg != nil { + // 'normal' cleanup, remove all images from the group + vgErr := vg.RemoveVolume(ctx, volume) + if vgErr != nil { + log.ErrorLog( + ctx, + "failed to remove volume %q from volume group %q: %v", + volume, vg, vgErr) + } + } + + // free all allocated volumes + if volume != nil { + volume.Destroy(ctx) + } + } + + if vg != nil { + // the VG should always be deleted, volumes can only belong to a single VG + log.DebugLog(ctx, "removing temporary volume group %q", vg) + + vgErr := vg.Delete(ctx) + if vgErr != nil { + log.ErrorLog(ctx, "failed to remove temporary volume group %q: %v", vg, vgErr) + } + + // free the resources of the VolumeGroup + vg.Destroy(ctx) + } + }() + for i, id := range req.GetSourceVolumeIds() { + var vol types.Volume + vol, err = mgr.GetVolumeByID(ctx, id) + if err != nil { + return nil, status.Errorf( + codes.InvalidArgument, + "failed to find required volume %q for volume group snapshot %q: %s", + id, + vgsName, + err.Error()) + } + volumes[i] = vol + } + + log.DebugLog(ctx, "all %d Volumes for VolumeGroup %q have been found", len(volumes), vgsName) + + groupSnapshot, err = mgr.GetVolumeGroupSnapshotByName(ctx, vgsName) + if groupSnapshot != nil { + defer groupSnapshot.Destroy(ctx) + + csiVGS, csiErr := groupSnapshot.ToCSI(ctx) + if csiErr != nil { + return nil, status.Error(codes.Aborted, csiErr.Error()) + } + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: csiVGS, + }, nil + } + if err != nil { + log.DebugLog(ctx, "need to create new volume group snapshot, "+ + "failed to get existing one with name %q: %v", vgsName, err) + } + + // create a temporary VolumeGroup with a different name + vg, err = mgr.CreateVolumeGroup(ctx, vgName) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to create volume group %q: %s", + vgName, + err.Error()) + } + // vg.Destroy(ctx) is called in a defer further up ^ + + log.DebugLog(ctx, "VolumeGroup %q has been created: %+v", vgName, vg) + + // add images to the group + for _, volume := range volumes { + err = vg.AddVolume(ctx, volume) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) + } + } + + groupSnapshot, err = mgr.CreateVolumeGroupSnapshot(ctx, vg, vgsName) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) + } + defer groupSnapshot.Destroy(ctx) + + csiVGS, err := groupSnapshot.ToCSI(ctx) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) + } + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: csiVGS, + }, nil +} + +func (cs *ControllerServer) DeleteVolumeGroupSnapshot( + ctx context.Context, + req *csi.DeleteVolumeGroupSnapshotRequest, +) (*csi.DeleteVolumeGroupSnapshotResponse, error) { + // FIXME: more checking of the request in needed + // 1. verify that all snapshots in the request are all snapshots in the group + // 2. delete the group + + mgr := NewManager(cs.Driver.GetInstanceID(), nil, req.GetSecrets()) + defer mgr.Destroy(ctx) + + groupSnapshot, err := mgr.GetVolumeGroupSnapshotByID(ctx, req.GetGroupSnapshotId()) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to get volume group snapshot with id %q: %v", + req.GetGroupSnapshotId(), err) + } + defer groupSnapshot.Destroy(ctx) + + err = groupSnapshot.Delete(ctx) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to delete volume group snapshot %q: %v", + groupSnapshot, err) + } + + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil +} + +// GetVolumeGroupSnapshot is sortof optional, only used for +// static/pre-provisioned VolumeGroupSnapshots. +func (cs *ControllerServer) GetVolumeGroupSnapshot( + ctx context.Context, + req *csi.GetVolumeGroupSnapshotRequest, +) (*csi.GetVolumeGroupSnapshotResponse, error) { + mgr := NewManager(cs.Driver.GetInstanceID(), nil, req.GetSecrets()) + defer mgr.Destroy(ctx) + + groupSnapshot, err := mgr.GetVolumeGroupSnapshotByID(ctx, req.GetGroupSnapshotId()) + if err != nil { + return nil, status.Errorf( + codes.Internal, + "failed to get volume group snapshot with id %q: %v", + req.GetGroupSnapshotId(), err) + } + defer groupSnapshot.Destroy(ctx) + + csiVGS, err := groupSnapshot.ToCSI(ctx) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) + } + + return &csi.GetVolumeGroupSnapshotResponse{ + GroupSnapshot: csiVGS, + }, nil +} From 42cb4b99b536c87f13310129e8098533c25d710b Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 13 Sep 2024 12:16:01 +0200 Subject: [PATCH 15/16] rbd: expose the GroupControllerService When the GroupSnapGetInfo go-ceph function is supported by librbd, the Group Controller Servive and VolumeGroupSnapshot capabilities can be exposed to the Container Orchestrator. Signed-off-by: Niels de Vos --- internal/rbd/driver/driver.go | 15 ++++++++++ internal/rbd/identityserver.go | 50 ++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/internal/rbd/driver/driver.go b/internal/rbd/driver/driver.go index 0bf6cfe04b5..4377d5e7629 100644 --- a/internal/rbd/driver/driver.go +++ b/internal/rbd/driver/driver.go @@ -25,6 +25,7 @@ import ( csiaddons "github.com/ceph/ceph-csi/internal/csi-addons/server" csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/rbd" + "github.com/ceph/ceph-csi/internal/rbd/features" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/k8s" "github.com/ceph/ceph-csi/internal/util/log" @@ -123,6 +124,19 @@ func (r *Driver) Run(conf *util.Config) { csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER, csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER, }) + + // GroupSnapGetInfo is used within the VolumeGroupSnapshot implementation + vgsSupported, vgsErr := features.SupportsGroupSnapGetInfo() + if vgsSupported { + r.cd.AddGroupControllerServiceCapabilities([]csi.GroupControllerServiceCapability_RPC_Type{ + csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT, + }) + } else { + log.DefaultLog("not enabling VolumeGroupSnapshot service capability") + } + if vgsErr != nil { + log.ErrorLogMsg("failed detecting VolumeGroupSnapshot support: %v", vgsErr) + } } if k8s.RunsOnKubernetes() && conf.IsNodeServer { @@ -178,6 +192,7 @@ func (r *Driver) Run(conf *util.Config) { IS: r.ids, CS: r.cs, NS: r.ns, + GS: r.cs, } s.Start(conf.Endpoint, srv, csicommon.MiddlewareServerOptionConfig{ LogSlowOpInterval: conf.LogSlowOpInterval, diff --git a/internal/rbd/identityserver.go b/internal/rbd/identityserver.go index 34691c201b4..ef67f27fcf0 100644 --- a/internal/rbd/identityserver.go +++ b/internal/rbd/identityserver.go @@ -20,6 +20,7 @@ import ( "context" csicommon "github.com/ceph/ceph-csi/internal/csi-common" + "github.com/ceph/ceph-csi/internal/rbd/features" "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -35,29 +36,44 @@ func (is *IdentityServer) GetPluginCapabilities( ctx context.Context, req *csi.GetPluginCapabilitiesRequest, ) (*csi.GetPluginCapabilitiesResponse, error) { - return &csi.GetPluginCapabilitiesResponse{ - Capabilities: []*csi.PluginCapability{ - { - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, - }, + caps := []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, }, }, - { - Type: &csi.PluginCapability_VolumeExpansion_{ - VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ - Type: csi.PluginCapability_VolumeExpansion_ONLINE, - }, + }, + { + Type: &csi.PluginCapability_VolumeExpansion_{ + VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ + Type: csi.PluginCapability_VolumeExpansion_ONLINE, }, }, - { - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, - }, + }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, }, }, }, + } + + // GroupSnapGetInfo is used within the VolumeGroupSnapshot implementation + vgsSupported, err := features.SupportsGroupSnapGetInfo() + if err == nil && vgsSupported { + gcs := csi.PluginCapability{ + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_GROUP_CONTROLLER_SERVICE, + }, + }, + } + caps = append(caps, &gcs) + } + + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: caps, }, nil } From 1686dbb900ec2a515af87818d882212043462b32 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 25 Oct 2024 17:24:50 +0200 Subject: [PATCH 16/16] rbd: set SnapshotGroupID on each Snapshot of a VolumeGroupSnapshot Without the SnapshotGroupID in the Snapshot object, Kubernetes CSI does not know that the Snapshot belongs to a group. In that case, it allows the deletion of the Snapshot, which should be denied. Signed-off-by: Niels de Vos --- internal/rbd/group/group_snapshot.go | 6 +++++ internal/rbd/manager.go | 5 ---- internal/rbd/rbd_util.go | 12 ++++++++++ internal/rbd/snapshot.go | 34 ++++++++++++++++++++++++---- internal/rbd/types/snapshot.go | 5 ++++ 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/internal/rbd/group/group_snapshot.go b/internal/rbd/group/group_snapshot.go index 7abb237addd..f565a600ac0 100644 --- a/internal/rbd/group/group_snapshot.go +++ b/internal/rbd/group/group_snapshot.go @@ -155,6 +155,12 @@ func NewVolumeGroupSnapshot( } volumeMap[handle] = name + + // store the CSI ID of the group in the snapshot attributes + snapErr = snapshot.SetVolumeGroup(ctx, creds, vgs.id) + if snapErr != nil { + return nil, fmt.Errorf("failed to set volume group ID %q for snapshot %q: %w", vgs.id, name, snapErr) + } } j, err := vgs.getJournal(ctx) diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go index 73f783fb027..8b0d6a06448 100644 --- a/internal/rbd/manager.go +++ b/internal/rbd/manager.go @@ -212,11 +212,6 @@ func (mgr *rbdManager) GetSnapshotByID(ctx context.Context, id string) (types.Sn } } - // FIXME: The snapshot will have RbdImageName set to the image that was - // used as source. This is not correct for group snapshots images, and - // need to be fixed. See rbdVolume.NewSnapshotByID() for more details. - snapshot.RbdImageName = snapshot.RbdSnapName - return snapshot, nil } diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index b76187aa0f1..fbae7f4181d 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -191,6 +191,9 @@ type rbdSnapshot struct { // RbdSnapName is the name of the RBD snapshot backing this rbdSnapshot SourceVolumeID string RbdSnapName string + + // groupID is the CSI volume group ID where this snapshot belongs to + groupID string } // imageFeature represents required image features and value. @@ -1054,6 +1057,15 @@ func genSnapFromSnapID( } } + if imageAttributes.GroupID != "" { + rbdSnap.groupID = imageAttributes.GroupID + // FIXME: The snapshot will have RbdImageName set to the image + // that was used as source. This is not correct for group + // snapshots images, and need to be fixed. See + // rbdVolume.NewSnapshotByID() for more details. + rbdSnap.RbdImageName = rbdSnap.RbdSnapName + } + err = rbdSnap.Connect(cr) if err != nil { return rbdSnap, fmt.Errorf("failed to connect to %q: %w", diff --git a/internal/rbd/snapshot.go b/internal/rbd/snapshot.go index 209299e5c50..6a06cefc288 100644 --- a/internal/rbd/snapshot.go +++ b/internal/rbd/snapshot.go @@ -153,11 +153,12 @@ func (rbdSnap *rbdSnapshot) ToCSI(ctx context.Context) (*csi.Snapshot, error) { } return &csi.Snapshot{ - SizeBytes: rbdSnap.VolSize, - SnapshotId: rbdSnap.VolID, - SourceVolumeId: rbdSnap.SourceVolumeID, - CreationTime: timestamppb.New(*created), - ReadyToUse: true, + SizeBytes: rbdSnap.VolSize, + SnapshotId: rbdSnap.VolID, + SourceVolumeId: rbdSnap.SourceVolumeID, + CreationTime: timestamppb.New(*created), + ReadyToUse: true, + GroupSnapshotId: rbdSnap.groupID, }, nil } @@ -315,3 +316,26 @@ func (rv *rbdVolume) NewSnapshotByID( return snap, nil } + +func (rbdSnap *rbdSnapshot) SetVolumeGroup(ctx context.Context, cr *util.Credentials, groupID string) error { + vi := util.CSIIdentifier{} + err := vi.DecomposeCSIID(rbdSnap.VolID) + if err != nil { + return err + } + + j, err := snapJournal.Connect(rbdSnap.Monitors, rbdSnap.RadosNamespace, cr) + if err != nil { + return fmt.Errorf("snapshot %q failed to connect to journal: %w", rbdSnap, err) + } + defer j.Destroy() + + err = j.StoreGroupID(ctx, rbdSnap.Pool, vi.ObjectUUID, groupID) + if err != nil { + return fmt.Errorf("failed to set volume group ID for snapshot %q: %w", rbdSnap, err) + } + + rbdSnap.groupID = groupID + + return nil +} diff --git a/internal/rbd/types/snapshot.go b/internal/rbd/types/snapshot.go index 90e4ef13e79..95487821618 100644 --- a/internal/rbd/types/snapshot.go +++ b/internal/rbd/types/snapshot.go @@ -21,6 +21,8 @@ import ( "time" "github.com/container-storage-interface/spec/lib/go/csi" + + "github.com/ceph/ceph-csi/internal/util" ) type Snapshot interface { @@ -35,4 +37,7 @@ type Snapshot interface { ToCSI(ctx context.Context) (*csi.Snapshot, error) GetCreationTime(ctx context.Context) (*time.Time, error) + + // SetVolumeGroup sets the CSI volume group ID in the snapshot. + SetVolumeGroup(ctx context.Context, creds *util.Credentials, vgID string) error }