diff --git a/Makefile b/Makefile index 3d984dcfa684..05efc6cef118 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ e2e.test: check-env .PHONY: rbd-group-snapshot rbd-group-snapshot: - go build -o _output/rbd-group-snapshot ./tools/rbd-group-snapshot + go build $(GO_TAGS) -o _output/rbd-group-snapshot ./tools/rbd-group-snapshot # # Update the generated deploy/ files when the template changed. This requires diff --git a/deploy/cephcsi/image/Dockerfile b/deploy/cephcsi/image/Dockerfile index 552e8273d918..e8f32512333d 100644 --- a/deploy/cephcsi/image/Dockerfile +++ b/deploy/cephcsi/image/Dockerfile @@ -68,6 +68,7 @@ COPY . ${SRC_DIR} # Build executable RUN make cephcsi +#RUN make rbd-group-snapshot #-- Final container FROM updated_base @@ -80,6 +81,7 @@ LABEL maintainers="Ceph-CSI Authors" \ description="Ceph-CSI Plugin" COPY --from=builder ${SRC_DIR}/_output/cephcsi /usr/local/bin/cephcsi +COPY --from=builder ${SRC_DIR}/_output/rbd-group-snapshot /usr/local/bin/rbd-group-snapshot # verify that all dynamically linked libraries are available RUN [ $(ldd /usr/local/bin/cephcsi | grep -c '=> not found') = '0' ] diff --git a/tools/rbd-group-snapshot/main.go_ b/tools/rbd-group-snapshot/main.go_ new file mode 100644 index 000000000000..2c6266f1a8f5 --- /dev/null +++ b/tools/rbd-group-snapshot/main.go_ @@ -0,0 +1,302 @@ +package main + +import ( + "fmt" + + "github.com/ceph/go-ceph/rados" + "github.com/ceph/go-ceph/rbd" +) + +var ( + imageNames = []string{ + "first-volume", + "second-volume", + } + + restoreName = "restored-image" + + pool = "ocs-storagecluster-cephblockpool" + + group = "all-the-volumes" + groupSnap = "all-the-snapshots" +) + +type rbdGroupTest struct { + conn *rados.Conn + ioctx *rados.IOContext +} + +func main() { + rgt := &rbdGroupTest{} + + rgt.connect() + defer rgt.conn.Shutdown() + + rgt.createImages() + defer rgt.removeImages() + + rgt.createGroup() + defer rgt.removeGroup() + + rgt.addImagesToGroup() + defer rgt.removeImagesFromGroup() + + rgt.createGroupSnapshot() + defer rgt.removeGroupSnapshot() + + fmt.Println("images are still in the group") + rgt.listSnapshots() + + rgt.listGroupSnapshot() + + rgt.removeImagesFromGroup() + + fmt.Println("images have been removed from the group") + rgt.listSnapshots() + + rgt.removeGroup() // fails as there is still a group snapshot? + + fmt.Println("the group has been removed - expected to fail") + rgt.listSnapshots() + + fmt.Println("the group snapshot has been removed") + rgt.removeGroupSnapshot() + + rgt.listSnapshots() + + // rgt.restoreFromSnapshot() + // defer rgt.removeRestoredImage() +} + +func (rgt *rbdGroupTest) connect() { + conn, err := rados.NewConn() + if err != nil { + panic(err) + } + + err = conn.ReadDefaultConfigFile() + if err != nil { + panic(err) + } + + err = conn.Connect() + if err != nil { + panic(err) + } + + rgt.conn = conn + + ioctx, err := conn.OpenIOContext(pool) + if err != nil { + panic(err) + } + + rgt.ioctx = ioctx +} + +func (rgt *rbdGroupTest) createImages() { + for _, name := range imageNames { + _, err := rbd.Create(rgt.ioctx, name, uint64(1<<22), 22) + if err != nil { + panic(err) + } + } +} + +func (rgt *rbdGroupTest) removeImages() { + fmt.Println("removing the images") + + for _, name := range imageNames { + err := rbd.RemoveImage(rgt.ioctx, name) + if err != nil { + fmt.Printf("failed to remove image %q: %v\n", name, err) + } + } +} + +func (rgt *rbdGroupTest) createGroup() { + err := rbd.GroupCreate(rgt.ioctx, group) + if err != nil { + panic(err) + } +} + +func (rgt *rbdGroupTest) removeGroup() { + fmt.Println("removing the group") + + err := rbd.GroupRemove(rgt.ioctx, group) + if err != nil { + fmt.Printf("failed to remove group %q: %v\n", group, err) + } +} + +func (rgt *rbdGroupTest) addImagesToGroup() { + for _, name := range imageNames { + err := rbd.GroupImageAdd(rgt.ioctx, group, rgt.ioctx, name) + if err != nil { + panic(err) + } + } +} + +func (rgt *rbdGroupTest) removeImagesFromGroup() { + fmt.Println("removing images from the group") + + for _, name := range imageNames { + err := rbd.GroupImageRemove(rgt.ioctx, group, rgt.ioctx, name) + if err != nil { + fmt.Printf("failed to remove image %q from group %q: %v\n", name, group, err) + } + } +} + +func (rgt *rbdGroupTest) createGroupSnapshot() { + err := rbd.GroupSnapCreate(rgt.ioctx, group, groupSnap) + if err != nil { + panic(err) + } +} + +func (rgt *rbdGroupTest) removeGroupSnapshot() { + fmt.Println("removing the group snapshot") + + err := rbd.GroupSnapRemove(rgt.ioctx, group, groupSnap) + if err != nil { + fmt.Printf("failed to remove group snapshot %q: %v\n", groupSnap, err) + } +} + +func (rgt *rbdGroupTest) listGroupSnapshot() { + fmt.Printf("listing snapshots of group %q\n", group) + + info, err := rbd.GroupSnapGetInfo(rgt.ioctx, group, groupSnap) + if err != nil { + panic(fmt.Sprintf("failed to list snapshots of group %q: %v\n", group, err)) + } + + fmt.Printf("snapshots in the group snapshot %q:\n", group+"@"+info.Name) + for _, snap := range info.Snapshots { + fmt.Printf(" - %q from %+v\n", snap.Name+"@"+info.SnapName, snap) + } +} + +func (rgt *rbdGroupTest) listSnapshots() { + img, err := rbd.OpenImage(rgt.ioctx, imageNames[0], rbd.NoSnapshot) + if err != nil { + panic(err) + } + defer img.Close() + + snaps, err := img.GetSnapshotNames() + if err != nil { + panic(err) + } + + fmt.Printf("listing %d snapshots for image %q\n", len(snaps), imageNames[0]) + for _, snap := range snaps { + fmt.Printf("Snapshot: %+v\n", snap) + } +} + +func (rgt *rbdGroupTest) restoreFromSnapshot() { + img, err := rbd.OpenImage(rgt.ioctx, imageNames[0], rbd.NoSnapshot) + if err != nil { + panic(err) + } + defer img.Close() + + snaps, err := img.GetSnapshotNames() + if err != nil { + panic(err) + } + + options := rbd.NewRbdImageOptions() + defer options.Destroy() + err = options.SetUint64(rbd.ImageOptionOrder, 22) + if err != nil { + panic(err) + } + // err = options.SetUint64(rbd.ImageOptionFeatures, 1) + // if err != nil { + // panic(err) + // } + + fmt.Printf("restoring image %q from parent %q at snapshot %q\n", restoreName, imageNames[0], snaps[0].Name) + snap := img.GetSnapshot(snaps[0].Name) + err = snap.Protect() + if err != nil { + panic(err) + } + defer snap.Unprotect() + + //err = rbd.CloneFromImage(img, snaps[0].Name, rgt.ioctx, restoreName, options) + err = rbd.CloneImageByID(rgt.ioctx, imageNames[0], snaps[0].Id, rgt.ioctx, restoreName, options) + if err != nil { + panic(err) + } + + /* + // alternative to the above -- segfaults, needs a snapshot + fmt.Printf("restoring image %q from parent %q without a snapshot\n", restoreName, imageNames[0]) + err = rbd.CloneFromImage(img, rbd.NoSnapshot, rgt.ioctx, restoreName, options) + if err != nil { + panic(err) + } + defer rbd.RemoveImage(rgt.ioctx, restoreName) + + restored, err := rbd.OpenImage(rgt.ioctx, restoreName, rbd.NoSnapshot) + if err != nil { + panic(err) + } + defer restored.Close() + + //err = restored.SetSnapshot(snaps[0].Name) + err = restored.SetSnapByID(snaps[0].Id) + if err != nil { + panic(err) + } + */ + + // alternative to the above + /* + snapname := "tmp-snap" + snap, err := img.CreateSnapshot(snapname) + if err != nil { + panic(err) + } + defer snap.Remove() + + err = snap.Protect() + if err != nil { + panic(err) + } + defer snap.Unprotect() + + fmt.Printf("restoring image %q from parent %q at snapshot %q\n", restoreName, imageNames[0], snapname) + err = rbd.CloneFromImage(img, snapname, rgt.ioctx, restoreName, options) + if err != nil { + panic(err) + } + defer rbd.RemoveImage(rgt.ioctx, restoreName) + + restored, err := rbd.OpenImage(rgt.ioctx, restoreName, rbd.NoSnapshot) + if err != nil { + panic(err) + } + defer restored.Close() + + err = restored.SetSnapByID(snaps[0].Id) + if err != nil { + panic(err) + } + */ +} + +func (rgt *rbdGroupTest) removeRestoredImage() { + fmt.Println("removing the restored image") + + err := rbd.RemoveImage(rgt.ioctx, restoreName) + if err != nil { + fmt.Printf("failed to remove image %q: %v\n", restoreName, err) + } +} diff --git a/tools/volume-group/main.go b/tools/volume-group/main.go new file mode 100644 index 000000000000..99052d94abe0 --- /dev/null +++ b/tools/volume-group/main.go @@ -0,0 +1,231 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + + "github.com/csi-addons/spec/lib/go/volumegroup" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +var ( + endpoint = "unix:///csi/csi-addons.sock" + + volumeGroupName = "the-group" + + parameters = map[string]string{ + "clusterID": "openshift-storage", + "pool": "ocs-storagecluster-cephblockpool", + } + + secrets = map[string]string{ + "userID": "csi-rbd-provisioner", + "userKey": "AQCTAqJmBumbBxAAF9UaIAw3VvjOy1mVkQJuKA==", + } + + volumes = []string{ + "0001-0011-openshift-storage-0000000000000002-0b5e14d8-2237-4262-a1d4-725a2942bf3a", + "0001-0011-openshift-storage-0000000000000002-a47a50fc-3bb1-40d9-bf58-9a522bfd112c", + } +) + +func main() { + conn, err := grpc.NewClient( + endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + fmt.Printf("failed to connect to %q: %v\n", endpoint, err) + + return + } + defer conn.Close() + + client := volumegroup.NewControllerClient(conn) + + fmt.Println("connected to socket " + endpoint) + + vgID, err := createVG(client) + if err != nil { + fmt.Println(err) + + return + } + + err = getVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = emptyVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = getVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = addVolumes(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = getVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = emptyVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + err = deleteVG(client, vgID) + if err != nil { + fmt.Println(err) + + return + } + + // VG should not exist anymore + err = getVG(client, vgID) + fmt.Printf("volume group with id %q should not exist anymore: %v\n", vgID, err) +} + +func createVG(client volumegroup.ControllerClient) (string, error) { + req := &volumegroup.CreateVolumeGroupRequest{ + Parameters: parameters, + Secrets: secrets, + + Name: volumeGroupName, + + // optional + VolumeIds: volumes, + } + + fmt.Println("sending request to controller") + + ctx := context.Background() + + res, err := client.CreateVolumeGroup(ctx, req) + if err != nil { + return "", fmt.Errorf("failed to create volume group %q: %w", req.GetName(), err) + } + + fmt.Printf("created volume group: %v\n", res) + + return res.GetVolumeGroup().GetVolumeGroupId(), nil +} + +func emptyVG(client volumegroup.ControllerClient, id string) error { + req := &volumegroup.ModifyVolumeGroupMembershipRequest{ + Secrets: secrets, + VolumeGroupId: id, + VolumeIds: nil, + } + + fmt.Println("sending request to controller") + + ctx := context.Background() + + res, err := client.ModifyVolumeGroupMembership(ctx, req) + if err != nil { + return fmt.Errorf("failed to modify volume group %q: %w", req.GetVolumeGroupId(), err) + } + + fmt.Printf("modified volume group: %v\n", res.GetVolumeGroup()) + + return nil +} + +func getVG(client volumegroup.ControllerClient, id string) error { + req := &volumegroup.ControllerGetVolumeGroupRequest{ + Secrets: secrets, + VolumeGroupId: id, + } + + fmt.Println("sending request to controller") + + ctx := context.Background() + + res, err := client.ControllerGetVolumeGroup(ctx, req) + if err != nil { + return fmt.Errorf("failed to get volume group %q: %w", req.GetVolumeGroupId(), err) + } + + fmt.Printf("got volume group: %v\n", res.GetVolumeGroup()) + + return nil +} + +func addVolumes(client volumegroup.ControllerClient, id string) error { + req := &volumegroup.ModifyVolumeGroupMembershipRequest{ + Secrets: secrets, + VolumeGroupId: id, + VolumeIds: volumes, + } + + fmt.Println("sending request to controller") + + ctx := context.Background() + + res, err := client.ModifyVolumeGroupMembership(ctx, req) + if err != nil { + return fmt.Errorf("failed to modify volume group %q: %w", req.GetVolumeGroupId(), err) + } + + fmt.Printf("added volumes to volume group: %v\n", res.GetVolumeGroup()) + + return nil +} + +func deleteVG(client volumegroup.ControllerClient, id string) error { + req := &volumegroup.DeleteVolumeGroupRequest{ + Secrets: secrets, + VolumeGroupId: id, + } + + fmt.Println("sending request to controller") + + ctx := context.Background() + + res, err := client.DeleteVolumeGroup(ctx, req) + if err != nil { + return fmt.Errorf("failed to delete volume group %q: %w", req.GetVolumeGroupId(), err) + } + + fmt.Printf("deleted volume group: %v\n", res) + + return nil +}