controller: fix frequent sidecar restarts (#12)

# Description Containers in the controller pod would frequently restart Upon closer inspection we found that only sidecar containers would restart with the same errors at roughly the same time. Judging by apiserver logs it looks like etcd throttled the requests because they were too frequent, which led the sidecars to restart. To fix this we double the lease durations, renew deadlines and retry periods for the leader elections of all sidecars. We also add some missing RBAC rules.
exoscale · Feb 20, 2024 · 78eaf5e · 78eaf5e
1 parent e526506
commit 78eaf5e
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+### Bug fixes
+
+* controller: fix frequent sidecar restarts #12 
+
 ## 0.29.1
 
 ### Improvements

diff --git a/deployment/controller-rbac.yaml b/deployment/controller-rbac.yaml
@@ -11,7 +11,7 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch", "delete"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims"]
     verbs: ["get", "list", "watch"]
@@ -35,7 +35,7 @@ rules:
     verbs: ["get", "list", "watch", "update", "create", "patch"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
+    verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -98,6 +98,9 @@ rules:
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions", "leases"]
     verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -123,6 +126,9 @@ rules:
   - apiGroups: [""]
     resources: ["pods", "events"]
     verbs: ["get", "watch", "list"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/deployment/controller.yaml b/deployment/controller.yaml
@@ -66,6 +66,9 @@ spec:
             - "--v=5"
             - "--csi-address=$(CSI_ADDRESS)"
             - "--leader-election"
+            - "--leader-election-lease-duration=30s"
+            - "--leader-election-renew-deadline=20s"
+            - "--leader-election-retry-period=10s"
             - "--feature-gates=Topology=true"
             - "--default-fstype=ext4"
           env:
@@ -87,6 +90,9 @@ spec:
             - "--v=5"
             - "--csi-address=$(CSI_ADDRESS)"
             - "--leader-election"
+            - "--leader-election-lease-duration=30s"
+            - "--leader-election-renew-deadline=20s"
+            - "--leader-election-retry-period=10s"
           env:
             - name: CSI_ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -106,6 +112,9 @@ spec:
             - "--v=5"
             - "--csi-address=$(CSI_ADDRESS)"
             - "--leader-election"
+            - "--leader-election-lease-duration=30s"
+            - "--leader-election-renew-deadline=20s"
+            - "--leader-election-retry-period=10s"
           env:
             - name: CSI_ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
@@ -124,6 +133,9 @@ spec:
           args:
             - "--v=5"
             - "--leader-election"
+            - "--leader-election-lease-duration=30s"
+            - "--leader-election-renew-deadline=20s"
+            - "--leader-election-retry-period=10s"
           resources:
             limits:
               cpu: 400m
@@ -137,6 +149,9 @@ spec:
             - "--v=5"
             - "--csi-address=$(CSI_ADDRESS)"
             - "--leader-election"
+            - "--leader-election-lease-duration=30s"
+            - "--leader-election-renew-deadline=20s"
+            - "--leader-election-retry-period=10s"
           env:
             - name: CSI_ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock

diff --git a/internal/integ/cluster/setup.go b/internal/integ/cluster/setup.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"time"
 
 	"github.com/exoscale/exoscale/csi-driver/internal/integ/flags"
 
@@ -150,5 +151,9 @@ func Setup() error {
 		}
 	}
 
+	// give the CSI some time to become ready
+	// TODO find a more appropriate way to do this.
+	time.Sleep(30 * time.Second)
+
 	return nil
 }
diff --git a/internal/integ/integ_test.go b/internal/integ/integ_test.go
@@ -63,7 +63,7 @@ func awaitExpectation(t *testing.T, expected interface{}, get getFunc) {
 	for i := 0; i < 10; i++ {
 		actual = get()
 
-		time.Sleep(5 * time.Second)
+		time.Sleep(10 * time.Second)
 
 		if assert.ObjectsAreEqualValues(expected, actual) {
 			break