diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 4e5bffc4..29b0d847 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -155,15 +155,13 @@ jobs: strategy: matrix: version: - - "4.1.4" + - "4.1.5" integration_test: # Single worker tests: - additional_serviceoptions - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job - # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # - timeout_prestop_termination # - upgrade_operator # See kind_311_tests job, Only works for 3.11 right now @@ -200,10 +198,17 @@ jobs: - scale_up - scale_up_stop_resume - seed_selection + - smoke_test_read_only_fs #- config_fql # OSS only - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA - # let other tests continue to run + include: + - version: 4.1.5 + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 # Modified version of cass-management-api + serverType: cassandra + integration_test: "smoke_test_read_only_fs" + + # let other tests continue to run # even if one fails fail-fast: false runs-on: ubuntu-latest diff --git a/.github/workflows/workflow-integration-tests.yaml b/.github/workflows/workflow-integration-tests.yaml index 4f3d3333..bb36d1e7 100644 --- a/.github/workflows/workflow-integration-tests.yaml +++ b/.github/workflows/workflow-integration-tests.yaml @@ -176,7 +176,6 @@ jobs: - additional_volumes # - delete_node_terminated_container # This does not test any operator behavior - podspec_simple - # - smoke_test_oss # Converted to test_all_the_things, see below job # - smoke_test_dse # Converted to test_all_the_things, see below job # - terminate # test_all_things # - timeout_prestop_termination # This is testing a Kubernetes behavior, not interesting to us diff --git a/CHANGELOG.md b/CHANGELOG.md index 500e2432..6f324add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti * [FEATURE] [#646](https://github.com/k8ssandra/cass-operator/issues/646) Allow starting multiple parallel pods if they have already previously bootstrapped and not planned for replacement. Set annotation ``cassandra.datastax.com/allow-parallel-starts: true`` to enable this feature. * [ENHANCEMENT] [#648](https://github.com/k8ssandra/cass-operator/issues/648) Make MinReadySeconds configurable value in the Spec. * [ENHANCEMENT] [#184](https://github.com/k8ssandra/cass-operator/issues/349) Add CassandraDatacenter.Status fields as metrics also +* [ENHANCEMENT] [#199](https://github.com/k8ssandra/cass-operator/issues/199) If .spec.readOnlyRootFilesystem is set, run the cassandra container with readOnlyRootFilesystem. Also, modify the default SecurityContext to mention runAsNonRoot: true ## v1.21.1 diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 5a13b335..ec9aa20e 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -268,6 +268,10 @@ type CassandraDatacenterSpec struct { // MinReadySeconds sets the minimum number of seconds for which a newly created pod should be ready without any of its containers crashing, for it to be considered available. Defaults to 5 seconds and is set in the StatefulSet spec. // Setting to 0 might cause multiple Cassandra pods to restart at the same time despite PodDisruptionBudget settings. MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + + // ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the + // new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) + ReadOnlyRootFilesystem *bool `json:"readOnlyRootFilesystem,omitempty"` } type NetworkingConfig struct { diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index ad4af351..6d091ed4 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -362,6 +362,11 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { *out = new(int32) **out = **in } + if in.ReadOnlyRootFilesystem != nil { + in, out := &in.ReadOnlyRootFilesystem, &out.ReadOnlyRootFilesystem + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CassandraDatacenterSpec. diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index 9cda1de7..2d7a9493 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -8820,6 +8820,11 @@ spec: - name type: object type: array + readOnlyRootFilesystem: + description: |- + ReadOnlyRootFilesystem makes the cassandra container to be run with a read-only root filesystem. Currently only functional when used with the + new k8ssandra-client config builder (Cassandra 4.1 and newer and HCD) + type: boolean replaceNodes: description: Deprecated Use CassandraTask replacenode to achieve correct node replacement. A list of pod names that need to be replaced. diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 56e5397a..2dfc83e7 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) const ( @@ -303,8 +304,27 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe EmptyDir: &corev1.EmptyDirVolumeSource{}, }, } + volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs} + if readOnlyFs(dc) { + tmp := corev1.Volume{ + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + etcCass := corev1.Volume{ + Name: "etc-cassandra", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + volumeDefaults = append(volumeDefaults, tmp, etcCass) + } + if dc.UseClientImage() { vBaseConfig := corev1.Volume{ Name: "server-config-base", @@ -435,7 +455,7 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl configMounts = append(configMounts, configBaseMount) - // Similar to k8ssandra 1.x, use config-container if use new config-builder replacement + // Similar to k8ssandra 1.x, use config-container if we use k8ssandra-client to build configs if configContainerIndex < 0 { configContainer = &corev1.Container{ Name: ServerBaseConfigContainerName, @@ -629,6 +649,12 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } + if readOnlyFs(dc) { + cassContainer.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: ptr.To[bool](true), + } + } + // Combine env vars envDefaults := []corev1.EnvVar{ @@ -636,6 +662,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla {Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")}, {Name: "DS_LICENSE", Value: "accept"}, {Name: "USE_MGMT_API", Value: "true"}, + {Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"}, {Name: "MGMT_API_EXPLICIT_START", Value: "true"}, } @@ -653,6 +680,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}) } + if readOnlyFs(dc) { + envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}) + } + cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env) // Combine ports @@ -706,6 +737,17 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } + if readOnlyFs(dc) { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "tmp", + MountPath: "/tmp", + }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "etc-cassandra", + MountPath: "/etc/cassandra", + }) + } + volumeMounts = combineVolumeMountSlices(volumeMounts, cassContainer.VolumeMounts) cassContainer.VolumeMounts = combineVolumeMountSlices(volumeMounts, generateStorageConfigVolumesMount(dc)) @@ -763,6 +805,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla return nil } +func readOnlyFs(dc *api.CassandraDatacenter) bool { + return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() +} + func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy() @@ -795,9 +841,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI if baseTemplate.Spec.SecurityContext == nil { var userID int64 = 999 baseTemplate.Spec.SecurityContext = &corev1.PodSecurityContext{ - RunAsUser: &userID, - RunAsGroup: &userID, - FSGroup: &userID, + RunAsUser: &userID, + RunAsGroup: &userID, + FSGroup: &userID, + RunAsNonRoot: ptr.To[bool](true), } } diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index f8926f7e..c116a73e 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -10,6 +10,7 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "k8s.io/apimachinery/pkg/api/resource" @@ -433,6 +434,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -459,6 +461,7 @@ func TestCassandraContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) } func TestHCDContainerEnvVars(t *testing.T) { @@ -468,6 +471,7 @@ func TestHCDContainerEnvVars(t *testing.T) { useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} hcdAutoConf := corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} templateSpec := &corev1.PodTemplateSpec{} dc := &api.CassandraDatacenter{ @@ -494,6 +498,7 @@ func TestHCDContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, hcdAutoConf)) } @@ -503,6 +508,7 @@ func TestDSEContainerEnvVars(t *testing.T) { nodeNameEnvVar := corev1.EnvVar{Name: "NODE_NAME", ValueFrom: selectorFromFieldPath("spec.nodeName")} useMgmtApiEnvVar := corev1.EnvVar{Name: "USE_MGMT_API", Value: "true"} explicitStartEnvVar := corev1.EnvVar{Name: "MGMT_API_EXPLICIT_START", Value: "true"} + noKeepAliveEnvVar := corev1.EnvVar{Name: "MGMT_API_NO_KEEP_ALIVE", Value: "true"} dseExplicitStartEnvVar := corev1.EnvVar{Name: "DSE_MGMT_EXPLICIT_START", Value: "true"} dseAutoConf := corev1.EnvVar{Name: "DSE_AUTO_CONF_OFF", Value: "all"} @@ -531,6 +537,7 @@ func TestDSEContainerEnvVars(t *testing.T) { assert.True(envVarsContains(cassContainer.Env, nodeNameEnvVar)) assert.True(envVarsContains(cassContainer.Env, useMgmtApiEnvVar)) assert.True(envVarsContains(cassContainer.Env, explicitStartEnvVar)) + assert.True(envVarsContains(cassContainer.Env, noKeepAliveEnvVar)) assert.True(envVarsContains(cassContainer.Env, dseAutoConf)) assert.True(envVarsContains(cassContainer.Env, dseExplicitStartEnvVar)) } @@ -1943,3 +1950,59 @@ func TestServiceAccountPrecedence(t *testing.T) { assert.Equal(test.accountName, pds.Spec.ServiceAccountName) } } + +func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "cassandra", + ServerVersion: "4.1.5", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "Unexpected containers containers received") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/etc/cassandra", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} diff --git a/pkg/reconciliation/construct_statefulset_test.go b/pkg/reconciliation/construct_statefulset_test.go index 45e02997..a0530b9e 100644 --- a/pkg/reconciliation/construct_statefulset_test.go +++ b/pkg/reconciliation/construct_statefulset_test.go @@ -436,9 +436,10 @@ func Test_newStatefulSetForCassandraPodSecurityContext(t *testing.T) { } defaultSecurityContext := &corev1.PodSecurityContext{ - RunAsUser: ptr.To(int64(999)), - RunAsGroup: ptr.To(int64(999)), - FSGroup: ptr.To(int64(999)), + RunAsUser: ptr.To(int64(999)), + RunAsGroup: ptr.To(int64(999)), + FSGroup: ptr.To(int64(999)), + RunAsNonRoot: ptr.To[bool](true), } tests := []struct { diff --git a/tests/smoke_test_dse/smoke_test_dse_suite_test.go b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go similarity index 71% rename from tests/smoke_test_dse/smoke_test_dse_suite_test.go rename to tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go index 1049c664..c1fa3de8 100644 --- a/tests/smoke_test_dse/smoke_test_dse_suite_test.go +++ b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go @@ -1,7 +1,7 @@ // Copyright DataStax, Inc. // Please see the included license file for details. -package smoke_test_dse +package smoke_test_read_only_fs import ( "fmt" @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" "github.com/k8ssandra/cass-operator/tests/kustomize" ginkgo_util "github.com/k8ssandra/cass-operator/tests/util/ginkgo" @@ -17,13 +16,12 @@ import ( ) var ( - testName = "Smoke test of basic functionality for one-node DSE cluster." - namespace = "test-smoke-test-dse" - dcName = "dc2" - dcYaml = "../testdata/smoke-test-dse.yaml" - dcResource = fmt.Sprintf("CassandraDatacenter/%s", dcName) - dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) - ns = ginkgo_util.NewWrapper(testName, namespace) + testName = "Smoke test of basic functionality for readOnlyRootFilesystem" + namespace = "test-smoke-test-read-only-fs" + dcName = "dc1" + dcYaml = "../testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml" + dcLabel = fmt.Sprintf("cassandra.datastax.com/datacenter=%s", dcName) + ns = ginkgo_util.NewWrapper(testName, namespace) ) func TestLifecycle(t *testing.T) { @@ -68,20 +66,6 @@ var _ = Describe(testName, func() { ns.WaitForDatacenterReady(dcName) ns.ExpectDoneReconciling(dcName) - step = "scale up to 2 nodes" - json = "{\"spec\": {\"size\": 2}}" - k = kubectl.PatchMerge(dcResource, json) - ns.ExecAndLog(step, k) - - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionTrue)) - ns.WaitForDatacenterOperatorProgress(dcName, "Updating", 60) - ns.WaitForDatacenterCondition(dcName, "ScalingUp", string(corev1.ConditionFalse)) - - // Ensure that when 'ScaleUp' becomes 'false' that our pods are in fact up and running - Expect(len(ns.GetDatacenterReadyPodNames(dcName))).To(Equal(2)) - - ns.WaitForDatacenterReady(dcName) - step = "deleting the dc" k = kubectl.DeleteFromFiles(dcYaml) ns.ExecAndLog(step, k) diff --git a/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml new file mode 100644 index 00000000..8a734a74 --- /dev/null +++ b/tests/testdata/default-single-rack-single-node-dc-with-readonly-fs.yaml @@ -0,0 +1,27 @@ +apiVersion: cassandra.datastax.com/v1beta1 +kind: CassandraDatacenter +metadata: + name: dc1 +spec: + clusterName: cluster1 + serverType: cassandra + serverVersion: "4.1.5" + serverImage: michaelburman290/cass-management-api:4.1.5-ubi8 + managementApiAuth: + insecure: {} + readOnlyRootFilesystem: true + size: 1 + storageConfig: + cassandraDataVolumeClaimSpec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + racks: + - name: r1 + config: + jvm-server-options: + initial_heap_size: "512m" + max_heap_size: "512m"