Add AdditionalAnnotations field and logic to reconcile it (#601)

* Add AdditionalAnnotations field and logic to reconcile it.

* Ensure rack reconciliation adds annotations when they are missing.

* Fix tests.

* Final test fixes.

* Update pod template spec annotations, not pod annotations.

* Michael's feedback.

* Revert changes to mergeInLabelsIfDifferent.

* Changelog.

CHANGELOG.md

@@ -11,6 +11,8 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
 
 ## unreleased
 
+* [FEATURE] [#601](https://github.com/k8ssandra/cass-operator/pull/601) Add additionalAnnotations field to CR so that all resources created by the operator can be annotated.
+
 ## v1.18.2
 
 * [BUGFIX] [#593](https://github.com/k8ssandra/cass-operator/issues/593) Update k8ssandra-client to 0.2.2 to fix the issue with clusterName config generation when using 4.1

apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -238,6 +238,9 @@ type CassandraDatacenterSpec struct {
 	// Additional Labels allows to define additional labels that will be included in all objects created by the operator. Note, user can override values set by default from the cass-operator and doing so could break cass-operator functionality.
 	AdditionalLabels map[string]string `json:"additionalLabels,omitempty"`
 
+	// Additional Annotations allows to define additional labels that will be included in all objects created by the operator. Note, user can override values set by default from the cass-operator and doing so could break cass-operator functionality.
+	AdditionalAnnotations map[string]string `json:"additionalAnnotations,omitempty"`
+
 	// CDC allows configuration of the change data capture agent which can run within the Management API container. Use it to send data to Pulsar.
 	CDC *CDCConfiguration `json:"cdc,omitempty"`
 

config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml

@@ -38,6 +38,14 @@ spec:
           spec:
             description: CassandraDatacenterSpec defines the desired state of a CassandraDatacenter
             properties:
+              additionalAnnotations:
+                additionalProperties:
+                  type: string
+                description: Additional Annotations allows to define additional labels
+                  that will be included in all objects created by the operator. Note,
+                  user can override values set by default from the cass-operator and
+                  doing so could break cass-operator functionality.
+                type: object
               additionalLabels:
                 additionalProperties:
                   type: string

internal/controllers/control/cassandratask_controller.go

@@ -238,6 +238,7 @@ func (r *CassandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 		utils.MergeMap(cassTask.Labels, dc.GetDatacenterLabels())
 		oplabels.AddOperatorLabels(cassTask.GetLabels(), dc)
+		oplabels.AddOperatorAnnotations(cassTask.GetAnnotations(), dc)
 
 		// Starting the run, set the Active label so we can quickly fetch the active ones
 		cassTask.GetLabels()[taskStatusLabel] = activeTaskLabelValue

pkg/oplabels/labels.go

@@ -32,6 +32,18 @@ func AddOperatorLabels(m map[string]string, dc *api.CassandraDatacenter) {
 			m[key] = api.CleanLabelValue(value)
 		}
 	}
+
+}
+
+func AddOperatorAnnotations(m map[string]string, dc *api.CassandraDatacenter) {
+	if m == nil {
+		m = make(map[string]string)
+	}
+	if len(dc.Spec.AdditionalAnnotations) != 0 {
+		for key, value := range dc.Spec.AdditionalAnnotations {
+			m[key] = value
+		}
+	}
 }
 
 func HasManagedByCassandraOperatorLabel(m map[string]string) bool {

pkg/reconciliation/construct_podtemplatespec.go

@@ -809,12 +809,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI
 
 	// Annotations
 
-	podAnnotations := map[string]string{}
-
 	if baseTemplate.Annotations == nil {
 		baseTemplate.Annotations = make(map[string]string)
 	}
-	baseTemplate.Annotations = utils.MergeMap(baseTemplate.Annotations, podAnnotations)
+	oplabels.AddOperatorAnnotations(baseTemplate.Annotations, dc)
 
 	// Affinity
 	nodeAffinityLabels, nodeAffinityLabelsConfigurationError := rackNodeAffinitylabels(dc, rack.Name)

pkg/reconciliation/construct_podtemplatespec_test.go

@@ -1003,24 +1003,34 @@ func TestCassandraDatacenter_buildContainers_additional_labels(t *testing.T) {
 			AdditionalLabels: map[string]string{
 				"Add": "Label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "Annotation",
+			},
 		},
 	}
 	dc.Spec.PodTemplateSpec.Labels = map[string]string{"abc": "123"}
 
 	spec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false)
-	got := spec.Labels
+	gotLabels := spec.Labels
 
-	expected := dc.GetRackLabels("testrack")
-	oplabels.AddOperatorLabels(expected, dc)
-	expected[api.CassNodeState] = stateReadyToStart
-	expected["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue
-	expected["abc"] = "123"
-	expected["Add"] = "Label"
+	expectedLabels := dc.GetRackLabels("testrack")
+	oplabels.AddOperatorLabels(expectedLabels, dc)
+	expectedLabels[api.CassNodeState] = stateReadyToStart
+	expectedLabels["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue
+	expectedLabels["abc"] = "123"
+	expectedLabels["Add"] = "Label"
 
 	assert.NoError(t, err, "should not have gotten error when building podTemplateSpec")
-	if !reflect.DeepEqual(expected, got) {
-		t.Errorf("labels = %v, want %v", got, expected)
+	if !reflect.DeepEqual(expectedLabels, gotLabels) {
+		t.Errorf("labels = %v, want %v", gotLabels, expectedLabels)
 	}
+
+	gotAnns := spec.Annotations
+	expectedAnnotations := map[string]string{"Add": "Annotation"}
+	if !reflect.DeepEqual(expectedAnnotations, gotAnns) {
+		t.Errorf("labels = %v, want %v", gotAnns, expectedAnnotations)
+	}
+
 }
 
 func TestCassandraDatacenter_buildPodTemplateSpec_overrideSecurityContext(t *testing.T) {

pkg/reconciliation/construct_service.go

@@ -70,6 +70,9 @@ func newServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Servi
 	}
 
 	service.Spec.Ports = ports
+	anns := make(map[string]string)
+	oplabels.AddOperatorAnnotations(anns, dc)
+	service.ObjectMeta.Annotations = anns
 
 	addAdditionalOptions(service, &dc.Spec.AdditionalServiceConfig.DatacenterService)
 
@@ -121,6 +124,10 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S
 	oplabels.AddOperatorLabels(labels, dc)
 	service.ObjectMeta.Labels = labels
 
+	anns := dc.GetAnnotations()
+	oplabels.AddOperatorAnnotations(anns, dc)
+	service.ObjectMeta.Annotations = anns
+
 	service.Spec.Selector = buildLabelSelectorForSeedService(dc)
 	service.Spec.PublishNotReadyAddresses = true
 
@@ -136,10 +143,13 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S
 func newAdditionalSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service {
 	labels := dc.GetDatacenterLabels()
 	oplabels.AddOperatorLabels(labels, dc)
+	anns := dc.GetAnnotations()
+	oplabels.AddOperatorAnnotations(anns, dc)
 	var service corev1.Service
 	service.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName()
 	service.ObjectMeta.Namespace = dc.Namespace
 	service.ObjectMeta.Labels = labels
+	service.ObjectMeta.Annotations = anns
 	// We omit the label selector because we will create the endpoints manually
 	service.Spec.Type = "ClusterIP"
 	service.Spec.ClusterIP = "None"
@@ -159,6 +169,9 @@ func newEndpointsForAdditionalSeeds(dc *api.CassandraDatacenter) (*corev1.Endpoi
 	endpoints.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName()
 	endpoints.ObjectMeta.Namespace = dc.Namespace
 	endpoints.ObjectMeta.Labels = labels
+	anns := dc.GetAnnotations()
+	oplabels.AddOperatorAnnotations(anns, dc)
+	endpoints.ObjectMeta.Annotations = anns
 
 	addresses := make([]corev1.EndpointAddress, 0, len(dc.Spec.AdditionalSeeds))
 	for _, additionalSeed := range dc.Spec.AdditionalSeeds {
@@ -290,5 +303,10 @@ func makeGenericHeadlessService(dc *api.CassandraDatacenter) *corev1.Service {
 	service.Spec.Selector = selector
 	service.Spec.Type = "ClusterIP"
 	service.Spec.ClusterIP = "None"
+
+	anns := make(map[string]string)
+	oplabels.AddOperatorAnnotations(anns, dc)
+	service.ObjectMeta.Annotations = anns
+
 	return &service
 }

pkg/reconciliation/construct_service_test.go

@@ -5,10 +5,11 @@ package reconciliation
 
 import (
 	"fmt"
-	"github.com/k8ssandra/cass-operator/pkg/oplabels"
 	"reflect"
 	"testing"
 
+	"github.com/k8ssandra/cass-operator/pkg/oplabels"
+
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -115,6 +116,9 @@ func TestLabelsWithNewSeedServiceForCassandraDatacenter(t *testing.T) {
 			AdditionalLabels: map[string]string{
 				"Add": "label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
 		},
 	}
 
@@ -134,6 +138,12 @@ func TestLabelsWithNewSeedServiceForCassandraDatacenter(t *testing.T) {
 	if !reflect.DeepEqual(expected, service.Labels) {
 		t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected)
 	}
+	if !reflect.DeepEqual(expected, service.Labels) {
+		t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{
+			"Add": "annotation",
+		})
+	}
+
 }
 
 func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) {
@@ -174,6 +184,9 @@ func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) {
 			AdditionalLabels: map[string]string{
 				"Add": "label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
 		},
 	}
 
@@ -194,6 +207,11 @@ func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) {
 	if !reflect.DeepEqual(expected, service.Labels) {
 		t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected)
 	}
+	if !reflect.DeepEqual(expected, service.Labels) {
+		t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{
+			"Add": "annotation",
+		})
+	}
 }
 
 func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) {
@@ -234,6 +252,9 @@ func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) {
 			AdditionalLabels: map[string]string{
 				"Add": "label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
 		},
 	}
 
@@ -255,6 +276,11 @@ func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) {
 	if !reflect.DeepEqual(expected, service.Labels) {
 		t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected)
 	}
+	if !reflect.DeepEqual(expected, service.Labels) {
+		t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{
+			"Add": "annotation",
+		})
+	}
 }
 
 func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) {
@@ -295,6 +321,9 @@ func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) {
 			AdditionalLabels: map[string]string{
 				"Add": "label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
 		},
 	}
 
@@ -315,6 +344,11 @@ func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) {
 	if !reflect.DeepEqual(expected, service.Labels) {
 		t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected)
 	}
+	if !reflect.DeepEqual(expected, service.Labels) {
+		t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{
+			"Add": "annotation",
+		})
+	}
 }
 
 func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T) {
@@ -355,6 +389,9 @@ func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T)
 			AdditionalLabels: map[string]string{
 				"Add": "label",
 			},
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
 		},
 	}
 
@@ -375,6 +412,11 @@ func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T)
 	if !reflect.DeepEqual(expected, service.Labels) {
 		t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected)
 	}
+	if !reflect.DeepEqual(expected, service.Labels) {
+		t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{
+			"Add": "annotation",
+		})
+	}
 }
 
 func TestAddingAdditionalLabels(t *testing.T) {
@@ -409,6 +451,25 @@ func TestAddingAdditionalLabels(t *testing.T) {
 	}
 }
 
+func TestAddingAdditionalAnnotations(t *testing.T) {
+	dc := &api.CassandraDatacenter{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "dc1",
+		},
+		Spec: api.CassandraDatacenterSpec{
+			ClusterName:   "piclem",
+			ServerVersion: "4.0.1",
+			AdditionalAnnotations: map[string]string{
+				"Add": "annotation",
+			},
+		},
+	}
+
+	service := newServiceForCassandraDatacenter(dc)
+
+	assert.Contains(t, service.Annotations, "Add")
+}
+
 func TestServicePorts(t *testing.T) {
 	tests := []struct {
 		name                string

pkg/reconciliation/construct_statefulset.go

@@ -78,6 +78,9 @@ func newStatefulSetForCassandraDatacenter(
 
 	statefulSetSelectorLabels := dc.GetRackLabels(rackName)
 
+	anns := make(map[string]string)
+	oplabels.AddOperatorAnnotations(anns, dc)
+
 	var volumeClaimTemplates []corev1.PersistentVolumeClaim
 
 	rack := dc.GetRack(rackName)
@@ -90,8 +93,9 @@ func newStatefulSetForCassandraDatacenter(
 
 	volumeClaimTemplates = []corev1.PersistentVolumeClaim{{
 		ObjectMeta: metav1.ObjectMeta{
-			Labels: pvcLabels,
-			Name:   PvcName,
+			Labels:      pvcLabels,
+			Name:        PvcName,
+			Annotations: anns,
 		},
 		Spec: *dc.Spec.StorageConfig.CassandraDataVolumeClaimSpec,
 	}}
@@ -100,8 +104,9 @@ func newStatefulSetForCassandraDatacenter(
 		if storage.PVCSpec != nil {
 			pvc := corev1.PersistentVolumeClaim{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:   storage.Name,
-					Labels: pvcLabels,
+					Name:        storage.Name,
+					Labels:      pvcLabels,
+					Annotations: anns,
 				},
 				Spec: *storage.PVCSpec,
 			}
@@ -126,9 +131,10 @@ func newStatefulSetForCassandraDatacenter(
 
 	result := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      nsName.Name,
-			Namespace: nsName.Namespace,
-			Labels:    statefulSetLabels,
+			Name:        nsName.Name,
+			Namespace:   nsName.Namespace,
+			Labels:      statefulSetLabels,
+			Annotations: anns,
 		},
 		Spec: appsv1.StatefulSetSpec{
 			Selector: &metav1.LabelSelector{
@@ -141,7 +147,6 @@ func newStatefulSetForCassandraDatacenter(
 			VolumeClaimTemplates: volumeClaimTemplates,
 		},
 	}
-	result.Annotations = map[string]string{}
 
 	if sts != nil && sts.Spec.ServiceName != "" && sts.Spec.ServiceName != result.Spec.ServiceName {
 		result.Spec.ServiceName = sts.Spec.ServiceName