From cce8716881f13c187245a553bf9feac9d80cd3b8 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 29 May 2024 15:49:51 +0300 Subject: [PATCH 1/6] Add new annotation, cassandra.datastax.com/readonly-fs that makes the cassandra container to run in ReadOnlyRootFilesystem securityContext --- .../construct_podtemplatespec.go | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 56e5397a..0d0ea922 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) const ( @@ -303,8 +304,27 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe EmptyDir: &corev1.EmptyDirVolumeSource{}, }, } + volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs} + if readOnlyFs(dc) { + tmp := corev1.Volume{ + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + etcCass := corev1.Volume{ + Name: "etc-cassandra", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + volumeDefaults = append(volumeDefaults, tmp, etcCass) + } + if dc.UseClientImage() { vBaseConfig := corev1.Volume{ Name: "server-config-base", @@ -435,7 +455,7 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl configMounts = append(configMounts, configBaseMount) - // Similar to k8ssandra 1.x, use config-container if use new config-builder replacement + // Similar to k8ssandra 1.x, use config-container if we use k8ssandra-client to build configs if configContainerIndex < 0 { configContainer = &corev1.Container{ Name: ServerBaseConfigContainerName, @@ -629,6 +649,12 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } + if readOnlyFs(dc) { + cassContainer.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: ptr.To[bool](true), + } + } + // Combine env vars envDefaults := []corev1.EnvVar{ @@ -706,6 +732,17 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } + if readOnlyFs(dc) { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "tmp", + MountPath: "/tmp", + }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "etc-cassandra", + MountPath: "/etc/cassandra", + }) + } + volumeMounts = combineVolumeMountSlices(volumeMounts, cassContainer.VolumeMounts) cassContainer.VolumeMounts = combineVolumeMountSlices(volumeMounts, generateStorageConfigVolumesMount(dc)) @@ -763,6 +800,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla return nil } +func readOnlyFs(dc *api.CassandraDatacenter) bool { + return metav1.HasAnnotation(dc.ObjectMeta, "cassandra.datastax.com/readonly-fs") +} + func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy() From e9db5f432fb4667412e4c951a9645bd1f34e1f8e Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 31 May 2024 18:38:44 +0300 Subject: [PATCH 2/6] Move from annotation to .spec.readOnlyRootFilesystem property, add runAsNonRoot to default securityContext --- CHANGELOG.md | 1 + apis/cassandra/v1beta1/cassandradatacenter_types.go | 4 ++++ .../cassandra.datastax.com_cassandradatacenters.yaml | 5 +++++ pkg/reconciliation/construct_podtemplatespec.go | 9 +++++---- tests/testdata/default-two-rack-two-node-dc.yaml | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 500e2432..6f324add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti * [FEATURE] [#646](https://github.com/k8ssandra/cass-operator/issues/646) Allow starting multiple parallel pods if they have already previously bootstrapped and not planned for replacement. Set annotation ``cassandra.datastax.com/allow-parallel-starts: true`` to enable this feature. * [ENHANCEMENT] [#648](https://github.com/k8ssandra/cass-operator/issues/648) Make MinReadySeconds configurable value in the Spec. * [ENHANCEMENT] [#184](https://github.com/k8ssandra/cass-operator/issues/349) Add CassandraDatacenter.Status fields as metrics also +* [ENHANCEMENT] [#199](https://github.com/k8ssandra/cass-operator/issues/199) If .spec.readOnlyRootFilesystem is set, run the cassandra container with readOnlyRootFilesystem. Also, modify the default SecurityContext to mention runAsNonRoot: true ## v1.21.1 diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 5a13b335..8f6539a9 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -268,6 +268,10 @@ type CassandraDatacenterSpec struct { // MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec. // Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings. MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + + // ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the + // new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) + ReadOnlyRootFilesystem bool `json:"readOnlyRootFilesystem,omitempty"` } type NetworkingConfig struct { diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index 9cda1de7..2d7a9493 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -8820,6 +8820,11 @@ spec: - name type: object type: array + readOnlyRootFilesystem: + description: |- + ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the + new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) + type: boolean replaceNodes: description: Deprecated Use CassandraTask replacenode to achieve correct node replacement. A list of pod names that need to be replaced. diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 0d0ea922..181f2186 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -801,7 +801,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } func readOnlyFs(dc *api.CassandraDatacenter) bool { - return metav1.HasAnnotation(dc.ObjectMeta, "cassandra.datastax.com/readonly-fs") + return dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() } func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { @@ -836,9 +836,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI if baseTemplate.Spec.SecurityContext == nil { var userID int64 = 999 baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsUser: &userID, - RunAsGroup: &userID, - FSGroup: &userID, + RunAsUser: &userID, + RunAsGroup: &userID, + FSGroup: &userID, + RunAsNonRoot: ptr.To[bool](true), } } diff --git a/tests/testdata/default-two-rack-two-node-dc.yaml b/tests/testdata/default-two-rack-two-node-dc.yaml index 97817e4c..8ed229a1 100644 --- a/tests/testdata/default-two-rack-two-node-dc.yaml +++ b/tests/testdata/default-two-rack-two-node-dc.yaml @@ -9,6 +9,7 @@ spec: serverVersion: "4.0.10" managementApiAuth: insecure: {} + readOnlyRootFilesystem: true size: 2 storageConfig: cassandraDataVolumeClaimSpec: From bb8e7332ba9871ec752629f846c2da9007532170 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 16 Jul 2024 14:49:15 +0300 Subject: [PATCH 3/6] Fix existing tests --- pkg/reconciliation/construct_statefulset_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/reconciliation/construct_statefulset_test.go b/pkg/reconciliation/construct_statefulset_test.go index 45e02997..a0530b9e 100644 --- a/pkg/reconciliation/construct_statefulset_test.go +++ b/pkg/reconciliation/construct_statefulset_test.go @@ -436,9 +436,10 @@ func Test_newStatefulSetForCassandraPodSecurityContext(t *testing.T) { } defaultSecurityContext := &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(999)), - RunAsGroup: ptr.To(int64(999)), - FSGroup: ptr.To(int64(999)), + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + FSGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To[bool](true), } tests := []struct { From e20b1fc5c3a9a246979ddf5980b8f8eeff588c89 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 16 Jul 2024 16:22:35 +0300 Subject: [PATCH 4/6] Add unit tests, e2e test, modify readOnlyRootFilesystem to be *bool (so we can set it to true by default later), add verification that MCAC is disabled when used with readOnlyRootFilesystem --- .github/workflows/kindIntegTest.yml | 13 ++-- .../workflows/workflow-integration-tests.yaml | 1 - .../v1beta1/cassandradatacenter_types.go | 2 +- .../v1beta1/zz_generated.deepcopy.go | 5 ++ config/manager/image_config.yaml | 2 +- config/manager/kustomization.yaml | 2 +- .../construct_podtemplatespec.go | 7 ++- .../construct_podtemplatespec_test.go | 63 +++++++++++++++++++ .../smoke_test_read_only_fs.go} | 30 +++------ ...-rack-single-node-dc-with-readonly-fs.yaml | 27 ++++++++ .../default-two-rack-two-node-dc.yaml | 1 - 11 files changed, 120 insertions(+), 33 deletions(-) rename tests/{smoke_test_dse/smoke_test_dse_suite_test.go => smoke_test_read_only_fs/smoke_test_read_only_fs.go} (71%) create mode 100644 tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 4e5bffc4..29b0d847 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -155,15 +155,13 @@ jobs: strategy: matrix: version: - - "4.1.4" + - "4.1.5" integration_test: # Single worker tests: - additional_serviceoptions - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job - # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # - timeout_prestop_termination # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now @@ -200,10 +198,17 @@ jobs: - scale_up - scale_up_stop_resume - seed_selection + - smoke_test_read_only_fs #- config_fql # OSS only - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA - # let other tests continue to run + include: + - version: 4.1.5 + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api + serverType: cassandra + integration_test: "smoke_test_read_only_fs" + + # let other tests continue to run # even if one fails fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/workflow-integration-tests.yaml b/.github/workflows/workflow-integration-tests.yaml index 4f3d3333..bb36d1e7 100644 --- a/.github/workflows/workflow-integration-tests.yaml +++ b/.github/workflows/workflow-integration-tests.yaml @@ -176,7 +176,6 @@ jobs: - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # test_all_things # - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 8f6539a9..ec9aa20e 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -271,7 +271,7 @@ type CassandraDatacenterSpec struct { // ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the // new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) - ReadOnlyRootFilesystem bool `json:"readOnlyRootFilesystem,omitempty"` + ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"` } type NetworkingConfig struct { diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index ad4af351..6d091ed4 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -362,6 +362,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = new(int32) **out = **in } + if in.ReadOnlyRootFilesystem != nil { + in, out := &in.ReadOnlyRootFilesystem, &out.ReadOnlyRootFilesystem + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CassandraDatacenterSpec. diff --git a/config/manager/image_config.yaml b/config/manager/image_config.yaml index 2d2de737..34904fb8 100644 --- a/config/manager/image_config.yaml +++ b/config/manager/image_config.yaml @@ -3,7 +3,7 @@ kind: ImageConfig metadata: name: image-config images: - system-logger: "k8ssandra/system-logger:latest" + system-logger: "k8ssandra/system-logger:v1.22.0-dev.e8bef6b-20240716" config-builder: "datastax/cass-config-builder:1.0-ubi8" k8ssandra-client: "k8ssandra/k8ssandra-client:v0.4.0" # cassandra: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 7621efda..b3dfc913 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,4 +14,4 @@ kind: Kustomization images: - name: controller newName: k8ssandra/cass-operator - newTag: latest + newTag: v1.22.0-dev.e8bef6b-20240716 diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 181f2186..84822762 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -662,6 +662,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla {Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}, {Name: "DS_LICENSE", Value: "accept"}, {Name: "USE_MGMT_API", Value: "true"}, + {Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}, {Name: "MGMT_API_EXPLICIT_START", Value: "true"}, } @@ -679,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}) } + if readOnlyFs(dc) { + envDefaults = append(envDefaults, corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"}) + } + cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env) // Combine ports @@ -801,7 +806,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } func readOnlyFs(dc *api.CassandraDatacenter) bool { - return dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() + return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() } func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index f8926f7e..711f3cd0 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -10,6 +10,7 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "k8s.io/apimachinery/pkg/api/resource" @@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) } func TestHCDContainerEnvVars(t *testing.T) { @@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) { useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, hcdAutoConf)) } @@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"} dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"} @@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, dseAutoConf)) assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar)) } @@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) { assert.Equal(test.accountName, pds.Spec.ServiceAccountName) } } + +func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "cassandra", + ServerVersion: "4.1.5", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "Unexpected containers containers received") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/etc/cassandra", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem + mcacDisabled := corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} diff --git a/tests/smoke_test_dse/smoke_test_dse_suite_test.go b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go similarity index 71% rename from tests/smoke_test_dse/smoke_test_dse_suite_test.go rename to tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go index 1049c664..c1fa3de8 100644 --- a/tests/smoke_test_dse/smoke_test_dse_suite_test.go +++ b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go @@ -1,7 +1,7 @@ // Copyright DataStax, Inc. // Please see the included license file for details. -package smoke_test_dse +package smoke_test_read_only_fs import ( "fmt" @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" "github.com/k8ssandra/cass-operator/tests/kustomize" ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo" @@ -17,13 +16,12 @@ import ( ) var ( - testName = "Smoke test of basic functionality for one-node DSE cluster." - namespace = "test-smoke-test-dse" - dcName = "dc2" - dcYaml = "../testdata/smoke-test-dse.yaml" - dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) - dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) - ns = ginkgo_util.NewWrapper(testName, namespace) + testName = "Smoke test of basic functionality for readOnlyRootFilesystem" + namespace = "test-smoke-test-read-only-fs" + dcName = "dc1" + dcYaml = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml" + dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) + ns = ginkgo_util.NewWrapper(testName, namespace) ) func TestLifecycle(t *testing.T) { @@ -68,20 +66,6 @@ var _ = Describe(testName, func() { ns.WaitForDatacenterReady(dcName) ns.ExpectDoneReconciling(dcName) - step = "scale up to 2 nodes" - json = "{\"spec\": {\"size\": 2}}" - k = kubectl.PatchMerge(dcResource, json) - ns.ExecAndLog(step, k) - - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue)) - ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60) - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse)) - - // Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running - Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2)) - - ns.WaitForDatacenterReady(dcName) - step = "deleting the dc" k = kubectl.DeleteFromFiles(dcYaml) ns.ExecAndLog(step, k) diff --git a/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml new file mode 100644 index 00000000..8a734a74 --- /dev/null +++ b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml @@ -0,0 +1,27 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc1 +spec: + clusterName: cluster1 + serverType: cassandra + serverVersion: "4.1.5" + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 + managementApiAuth: + insecure: {} + readOnlyRootFilesystem: true + size: 1 + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + racks: + - name: r1 + config: + jvm-server-options: + initial_heap_size: "512m" + max_heap_size: "512m" diff --git a/tests/testdata/default-two-rack-two-node-dc.yaml b/tests/testdata/default-two-rack-two-node-dc.yaml index 8ed229a1..97817e4c 100644 --- a/tests/testdata/default-two-rack-two-node-dc.yaml +++ b/tests/testdata/default-two-rack-two-node-dc.yaml @@ -9,7 +9,6 @@ spec: serverVersion: "4.0.10" managementApiAuth: insecure: {} - readOnlyRootFilesystem: true size: 2 storageConfig: cassandraDataVolumeClaimSpec: From c1a5a0ab775e9fc20c8302422d1835ce63fff4ee Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 16 Jul 2024 16:31:48 +0300 Subject: [PATCH 5/6] Remove $ character --- pkg/reconciliation/construct_podtemplatespec.go | 2 +- pkg/reconciliation/construct_podtemplatespec_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 84822762..2dfc83e7 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -681,7 +681,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } if readOnlyFs(dc) { - envDefaults = append(envDefaults, corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"}) + envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}) } cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env) diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index 711f3cd0..c116a73e 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -2003,6 +2003,6 @@ func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem - mcacDisabled := corev1.EnvVar{Name: "$MGMT_API_DISABLE_MCAC", Value: "true"} + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} assert.True(envVarsContains(containers[0].Env, mcacDisabled)) } From b7dbb30b4dd422e3f98a151c56036189f75690c5 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 19 Jul 2024 14:39:21 +0300 Subject: [PATCH 6/6] Revert config changes --- config/manager/image_config.yaml | 2 +- config/manager/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/image_config.yaml b/config/manager/image_config.yaml index 34904fb8..2d2de737 100644 --- a/config/manager/image_config.yaml +++ b/config/manager/image_config.yaml @@ -3,7 +3,7 @@ kind: ImageConfig metadata: name: image-config images: - system-logger: "k8ssandra/system-logger:v1.22.0-dev.e8bef6b-20240716" + system-logger: "k8ssandra/system-logger:latest" config-builder: "datastax/cass-config-builder:1.0-ubi8" k8ssandra-client: "k8ssandra/k8ssandra-client:v0.4.0" # cassandra: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index b3dfc913..7621efda 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,4 +14,4 @@ kind: Kustomization images: - name: controller newName: k8ssandra/cass-operator - newTag: v1.22.0-dev.e8bef6b-20240716 + newTag: latest