Added support for clickhouse and kafka user secret ref

wandb · Nov 27, 2024 · 318cae2 · 318cae2
1 parent ddb0dab
commit 318cae2
Show file tree

Hide file tree

Showing 10 changed files with 55 additions and 9 deletions.
diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock
@@ -48,4 +48,4 @@ dependencies:
   repository: file://charts/yace
   version: 0.1.0
 digest: sha256:c3cd39db4bfcb9f25b1e666b9786d28b5045d3337bfdd98a6d9a1be2c6b075b5
-generated: "2024-11-20T14:26:55.38318-08:00"
+generated: "2024-11-27T17:17:00.195546+05:30"
diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.19.2
+version: 0.19.4
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

diff --git a/charts/operator-wandb/charts/app/templates/_deployment.tpl b/charts/operator-wandb/charts/app/templates/_deployment.tpl
@@ -251,7 +251,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.kafka.passwordSecret" . }}
-                  key: KAFKA_CLIENT_PASSWORD
+                  key: {{ include "wandb.kafka.passwordSecret.passwordKey" .}}
             - name: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE
               value: {{ include "wandb.kafka.runUpdatesShadowTopic" .}}
             - name: KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS

diff --git a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml
@@ -60,7 +60,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
       {{- end }}
@@ -97,7 +97,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
           {{- if not .Values.datadog.enabled }}

diff --git a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml
@@ -61,7 +61,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
       restartPolicy: "Never"

diff --git a/charts/operator-wandb/templates/_clickhouse.tpl b/charts/operator-wandb/templates/_clickhouse.tpl
@@ -9,8 +9,23 @@ Return the kafka client password
 Return name of secret where clickhouse information is stored
 */}}
 {{- define "wandb.clickhouse.passwordSecret" -}}
-{{- print .Release.Name "-clickhouse" -}}
+{{- if .Values.global.clickhouse.passwordSecret.name }}
+  {{- .Values.global.clickhouse.passwordSecret.name -}}
+{{- else -}}
+  {{- print .Release.Name "-clickhouse" -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Return name of secret where clickhouse information is stored
+*/}}
+{{- define "wandb.clickhouse.passwordSecret.passwordKey" -}}
+{{- if .Values.global.clickhouse.passwordSecret.name }}
+  {{- .Values.global.clickhouse.passwordSecret.passwordKey -}}
+{{- else -}}
+  CLICKHOUSE_PASSWORD
 {{- end -}}
+{{- end }}
 
 {{/*
 Return the redis host

diff --git a/charts/operator-wandb/templates/_kafka.tpl b/charts/operator-wandb/templates/_kafka.tpl
@@ -31,8 +31,23 @@ Return the kafka client password
 Return name of secret where kafka information is stored
 */}}
 {{- define "wandb.kafka.passwordSecret" -}}
-{{- print .Release.Name "-kafka" -}}
+{{- if .Values.global.kafka.passwordSecret.name }}
+  {{- .Values.global.kafka.passwordSecret.name -}}
+{{- else -}}
+  {{- print .Release.Name "-kafka" -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Return name of secret where kafka information is stored
+*/}}
+{{- define "wandb.kafka.passwordSecret.passwordKey" -}}
+{{- if .Values.global.kafka.passwordSecret.name }}
+  {{- .Values.global.kafka.passwordSecret.passwordKey -}}
+{{- else -}}
+  KAFKA_CLIENT_PASSWORD
 {{- end -}}
+{{- end }}
 
 {{/*
 Return the kafka broker url port

diff --git a/charts/operator-wandb/templates/clickhouse.yaml b/charts/operator-wandb/templates/clickhouse.yaml
@@ -1,4 +1,5 @@
 ---
+{{- if not .Values.global.clickhouse.passwordSecret.name }}
 {{- $secretName := (include "wandb.clickhouse.passwordSecret" .)  }}
 apiVersion: v1
 kind: Secret
@@ -7,4 +8,5 @@ metadata:
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
-  CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }}
+  CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }}
+{{- end }}
diff --git a/charts/operator-wandb/templates/kafka.yaml b/charts/operator-wandb/templates/kafka.yaml
@@ -1,3 +1,4 @@
+{{- if not .Values.global.kafka.passwordSecret.name }}
 {{- $secretName := (include "wandb.kafka.passwordSecret" .)  }}
 apiVersion: v1
 kind: Secret
@@ -7,3 +8,4 @@ metadata:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
   KAFKA_CLIENT_PASSWORD: {{ include "wandb.kafka.password" . | b64enc }}
+{{- end }}
diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml
@@ -76,6 +76,9 @@ global:
     host: ""
     port: 8443
     password: "fake"
+    passwordSecret:
+      name: ""
+      passwordKey: "CLICKHOUSE_PASSWORD"
     database: "weave_trace_db"
     user: "default"
 
@@ -138,6 +141,13 @@ global:
     # the kafka dependency chart.
     user: &kafkaUser "wandb"
     password: &kafkaPassword "wandb"
+## The secret can be created using the following command:
+## "kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=controller-password=CONTROLLER_PASSWORD"
+## Replace SECRET_NAME, CLIENT_PASSWORD1, INTER_BROKER_PASSWORD, and CONTROLLER_PASSWORD with appropriate values.
+    passwordSecret:
+      name: &kafkaSecretName ""
+      # This (client-passwords) should match the key name used in the secret
+      passwordKey: "client-passwords"
     brokerHost: ""
     brokerPort: 9092
     runUpdatesShadowTopic: ""
@@ -365,6 +375,8 @@ kafka:
       users:
         - *kafkaUser
       passwords: *kafkaPassword
+    existingSecret: *kafkaSecretName
+
   kraft:
     # This field is a UUID. It is *strongly* recommended to supply a new UUID yourself for production installs.
     clusterId: "ffFF1H3AQKGdBnsqAbJKew"