Merge pull request #600 from makhov/etcd-auto-compaction

Etcd args and auto-compaction
k0sproject · Jun 4, 2024 · fb89716 · fb89716
2 parents 2f17df9 + 18191a8
commit fb89716
Show file tree

Hide file tree

Showing 12 changed files with 133 additions and 4 deletions.
diff --git a/api/k0smotron.io/v1beta1/k0smotroncluster_types.go b/api/k0smotron.io/v1beta1/k0smotroncluster_types.go
@@ -240,6 +240,9 @@ type EtcdSpec struct {
 	// Image defines the etcd image to be deployed.
 	//+kubebuilder:default="quay.io/k0sproject/etcd:v3.5.13"
 	Image string `json:"image"`
+	// Args defines the etcd arguments.
+	//+kubebuilder:validation:Optional
+	Args []string `json:"args,omitempty"`
 	// Persistence defines the persistence configuration.
 	//+kubebuilder:validation:Optional
 	Persistence EtcdPersistenceSpec `json:"persistence"`

diff --git a/api/k0smotron.io/v1beta1/zz_generated.deepcopy.go b/api/k0smotron.io/v1beta1/zz_generated.deepcopy.go
diff --git a/...g/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml b/...g/clusterapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml
@@ -76,6 +76,11 @@ spec:
                   persistence: {}
                 description: Etcd defines the etcd configuration.
                 properties:
+                  args:
+                    description: Args defines the etcd arguments.
+                    items:
+                      type: string
+                    type: array
                   image:
                     default: quay.io/k0sproject/etcd:v3.5.13
                     description: Image defines the etcd image to be deployed.

diff --git a/...rapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanetemplates.yaml b/...rapi/controlplane/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanetemplates.yaml
@@ -98,6 +98,11 @@ spec:
                           persistence: {}
                         description: Etcd defines the etcd configuration.
                         properties:
+                          args:
+                            description: Args defines the etcd arguments.
+                            items:
+                              type: string
+                            type: array
                           image:
                             default: quay.io/k0sproject/etcd:v3.5.13
                             description: Image defines the etcd image to be deployed.

diff --git a/config/clusterapi/k0smotron.io/bases/k0smotron.io_clusters.yaml b/config/clusterapi/k0smotron.io/bases/k0smotron.io_clusters.yaml
@@ -79,6 +79,11 @@ spec:
                   persistence: {}
                 description: Etcd defines the etcd configuration.
                 properties:
+                  args:
+                    description: Args defines the etcd arguments.
+                    items:
+                      type: string
+                    type: array
                   image:
                     default: quay.io/k0sproject/etcd:v3.5.13
                     description: Image defines the etcd image to be deployed.

diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanes.yaml
@@ -76,6 +76,11 @@ spec:
                   persistence: {}
                 description: Etcd defines the etcd configuration.
                 properties:
+                  args:
+                    description: Args defines the etcd arguments.
+                    items:
+                      type: string
+                    type: array
                   image:
                     default: quay.io/k0sproject/etcd:v3.5.13
                     description: Image defines the etcd image to be deployed.

diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_k0smotroncontrolplanetemplates.yaml
@@ -98,6 +98,11 @@ spec:
                           persistence: {}
                         description: Etcd defines the etcd configuration.
                         properties:
+                          args:
+                            description: Args defines the etcd arguments.
+                            items:
+                              type: string
+                            type: array
                           image:
                             default: quay.io/k0sproject/etcd:v3.5.13
                             description: Image defines the etcd image to be deployed.

diff --git a/config/crd/bases/k0smotron.io_clusters.yaml b/config/crd/bases/k0smotron.io_clusters.yaml
@@ -79,6 +79,11 @@ spec:
                   persistence: {}
                 description: Etcd defines the etcd configuration.
                 properties:
+                  args:
+                    description: Args defines the etcd arguments.
+                    items:
+                      type: string
+                    type: array
                   image:
                     default: quay.io/k0sproject/etcd:v3.5.13
                     description: Image defines the etcd image to be deployed.

diff --git a/docs/resource-reference.md b/docs/resource-reference.md
@@ -2437,6 +2437,13 @@ Etcd defines the etcd configuration.
             <i>Default</i>: quay.io/k0sproject/etcd:v3.5.13<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>args</b></td>
+        <td>[]string</td>
+        <td>
+          Args defines the etcd arguments.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b><a href="#k0smotroncontrolplanespecetcdpersistence">persistence</a></b></td>
         <td>object</td>
@@ -7309,6 +7316,13 @@ Etcd defines the etcd configuration.
             <i>Default</i>: quay.io/k0sproject/etcd:v3.5.13<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>args</b></td>
+        <td>[]string</td>
+        <td>
+          Args defines the etcd arguments.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b><a href="#k0smotroncontrolplanetemplatespectemplatespecetcdpersistence">persistence</a></b></td>
         <td>object</td>
@@ -28096,6 +28110,13 @@ Etcd defines the etcd configuration.
             <i>Default</i>: quay.io/k0sproject/etcd:v3.5.13<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>args</b></td>
+        <td>[]string</td>
+        <td>
+          Args defines the etcd arguments.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b><a href="#clusterspecetcdpersistence">persistence</a></b></td>
         <td>object</td>

diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd.go
@@ -17,9 +17,11 @@ limitations under the License.
 package k0smotronio
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"strings"
+	"text/template"
 
 	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
 
@@ -35,6 +37,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+var etcdEntrypointScriptTmpl *template.Template
+
+func init() {
+	etcdEntrypointScriptTmpl = template.Must(template.New("entrypoint.sh").Parse(etcdEntrypointScriptTemplate))
+}
+
 func (r *ClusterReconciler) reconcileEtcd(ctx context.Context, kmc *km.Cluster) error {
 	if kmc.Spec.KineDataSourceURL != "" || kmc.Spec.KineDataSourceSecretName != "" {
 		return nil
@@ -142,6 +150,13 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in
 		pvc.Spec.StorageClassName = &kmc.Spec.Etcd.Persistence.StorageClass
 	}
 
+	var etcdEntrypointScriptBuf bytes.Buffer
+	_ = etcdEntrypointScriptTmpl.Execute(&etcdEntrypointScriptBuf, struct {
+		Args []string
+	}{
+		Args: kmc.Spec.Etcd.Args,
+	})
+
 	statefulSet := apps.StatefulSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps/v1",
@@ -230,7 +245,7 @@ func (r *ClusterReconciler) generateEtcdStatefulSet(kmc *km.Cluster, replicas in
 						Image:           kmc.Spec.Etcd.Image,
 						ImagePullPolicy: v1.PullIfNotPresent,
 						Command:         []string{"/bin/bash"},
-						Args:            []string{"-c", entrypointScript},
+						Args:            []string{"-c", etcdEntrypointScriptBuf.String()},
 						Env: []v1.EnvVar{
 							{Name: "SVC_NAME", Value: kmc.GetEtcdServiceName()},
 							{Name: "ETCDCTL_ENDPOINTS", Value: fmt.Sprintf("https://%s:2379", kmc.GetEtcdServiceName())},
@@ -319,15 +334,14 @@ func calculateDesiredReplicas(kmc *km.Cluster) int32 {
 	return desiredReplicas
 }
 
-var entrypointScript = `
+const etcdEntrypointScriptTemplate = `
 
 export ETCD_INITIAL_CLUSTER_STATE="new"
 if [[ -f /var/lib/k0s/etcd/existing ]]; then
   export ETCD_INITIAL_CLUSTER_STATE="existing"
 fi
 
 etcd --name ${HOSTNAME} \
-  --data-dir=/var/lib/k0s/etcd \
   --listen-peer-urls=https://0.0.0.0:2380 \
   --listen-client-urls=https://0.0.0.0:2379 \
   --advertise-client-urls=https://${HOSTNAME}.${SVC_NAME}:2379 \
@@ -341,7 +355,14 @@ etcd --name ${HOSTNAME} \
   --peer-key-file=/var/lib/k0s/pki/etcd/peer.key \
   --peer-cert-file=/var/lib/k0s/pki/etcd/peer.crt \
   --peer-client-cert-auth=true \
-  --enable-pprof=false 
+  --enable-pprof=false \
+  --auto-compaction-mode=periodic \
+  --auto-compaction-retention=5m \
+  --snapshot-count=10000 \
+{{- range $arg := .Args }}
+  {{ $arg }} \
+{{- end }}
+  --data-dir=/var/lib/k0s/etcd 
 `
 
 var initEntryScript = `

diff --git a/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go b/internal/controller/k0smotron.io/k0smotroncluster_etcd_test.go
@@ -18,6 +18,7 @@ package k0smotronio
 
 import (
 	"github.com/stretchr/testify/assert"
+	"strings"
 	"testing"
 
 	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
@@ -43,3 +44,48 @@ func TestEtcd_calculateDesiredReplicas(t *testing.T) {
 		})
 	}
 }
+
+func TestEtcd_generateEtcdStatefulSet(t *testing.T) {
+	var tests = []struct {
+		cluster *km.Cluster
+		want    []string
+	}{
+		{
+			cluster: &km.Cluster{},
+			want: []string{
+				"--auto-compaction-mode=periodic",
+				"--auto-compaction-retention=5m",
+				"--snapshot-count=10000",
+			}},
+		{
+			cluster: &km.Cluster{Spec: km.ClusterSpec{Etcd: km.EtcdSpec{Args: []string{
+				"--auto-compaction-mode=periodic",
+			}}}},
+			want: []string{
+				"--auto-compaction-mode=periodic",
+				"--auto-compaction-retention=5m",
+				"--snapshot-count=10000",
+			}},
+		{
+			cluster: &km.Cluster{Spec: km.ClusterSpec{Etcd: km.EtcdSpec{Args: []string{
+				"--auto-compaction-mode=periodic",
+				"--auto-compaction-retention=2h",
+				"--snapshot-count=50000",
+			}}}},
+			want: []string{
+				"--auto-compaction-mode=periodic",
+				"--auto-compaction-retention=2h",
+				"--snapshot-count=50000",
+			}},
+	}
+
+	for _, tc := range tests {
+		t.Run("", func(t *testing.T) {
+			r := new(ClusterReconciler)
+			sts := r.generateEtcdStatefulSet(tc.cluster, 1)
+			for _, w := range tc.want {
+				assert.True(t, strings.Contains(sts.Spec.Template.Spec.Containers[0].Args[1], w))
+			}
+		})
+	}
+}
diff --git a/inttest/scaling-etcd/scaling_etcd_test.go b/inttest/scaling-etcd/scaling_etcd_test.go
@@ -190,6 +190,9 @@ var clusterResource = `
 				"spec": {
 					"telemetry": {"enabled": false}
 				}
+			},
+			"etcd": {
+				"args": ["--auto-compaction-retention=1h", "--snapshot-count=500000"]
 			}
 		}
 	  }