Merge branch 'main' into dacbd/redis-secret-name

wandb · Dec 20, 2024 · d501610 · d501610
2 parents 53683cd + 9177b04
commit d501610
Show file tree

Hide file tree

Showing 9 changed files with 60 additions and 10 deletions.
diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock
@@ -31,7 +31,7 @@ dependencies:
   version: 25.3.5
 - name: etcd
   repository: oci://registry-1.docker.io/bitnamicharts
-  version: 10.6.1
+  version: 10.7.1
 - name: bufstream
   repository: file://charts/bufstream
   version: 0.3.1
@@ -56,5 +56,5 @@ dependencies:
 - name: wandb-base
   repository: file://../wandb-base
   version: 0.1.1
-digest: sha256:1eb284a2d6588f3e3d90293abb99f0e25b02fd62a87758f023d809879a82e31d
-generated: "2024-12-06T11:08:12.69179-08:00"
+digest: sha256:3c02029e1921428d7f45866291a8c42b79894505ee47864c54f34fcd847b2793
+generated: "2024-12-19T11:00:55.672581+05:30"
diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml
@@ -257,8 +257,12 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.kafka.passwordSecret" . }}
-                  key: KAFKA_CLIENT_PASSWORD
+                  key: {{ include "wandb.kafka.passwordSecret.passwordKey" .}}
                   optional: true
+            - name: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE
+              value: {{ include "wandb.kafka.runUpdatesShadowTopic" .}}
+            - name: KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS
+              value: "{{ include "wandb.kafka.runUpdatesShadowNumPartitions" .}}"
             {{- end }}
             - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE
               value: >

diff --git a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml
@@ -60,7 +60,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
       {{- end }}
@@ -97,7 +97,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
           {{- if not .Values.datadog.enabled }}

diff --git a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml
@@ -61,7 +61,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ include "wandb.clickhouse.passwordSecret" . }}
-                  key: CLICKHOUSE_PASSWORD
+                  key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }}
             {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
       restartPolicy: "Never"

diff --git a/charts/operator-wandb/templates/_clickhouse.tpl b/charts/operator-wandb/templates/_clickhouse.tpl
@@ -9,8 +9,23 @@ Return the kafka client password
 Return name of secret where clickhouse information is stored
 */}}
 {{- define "wandb.clickhouse.passwordSecret" -}}
-{{- print .Release.Name "-clickhouse" -}}
+{{- if .Values.global.clickhouse.passwordSecret.name }}
+  {{- .Values.global.clickhouse.passwordSecret.name -}}
+{{- else -}}
+  {{- print .Release.Name "-clickhouse" -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Return name of secret where clickhouse information is stored
+*/}}
+{{- define "wandb.clickhouse.passwordSecret.passwordKey" -}}
+{{- if .Values.global.clickhouse.passwordSecret.name }}
+  {{- .Values.global.clickhouse.passwordSecret.passwordKey -}}
+{{- else -}}
+  CLICKHOUSE_PASSWORD
 {{- end -}}
+{{- end }}
 
 {{/*
 Return the redis host

diff --git a/charts/operator-wandb/templates/_kafka.tpl b/charts/operator-wandb/templates/_kafka.tpl
@@ -35,8 +35,23 @@ Return the kafka client password
 Return name of secret where kafka information is stored
 */}}
 {{- define "wandb.kafka.passwordSecret" -}}
-{{- print .Release.Name "-kafka" -}}
+{{- if .Values.global.kafka.passwordSecret.name }}
+  {{- .Values.global.kafka.passwordSecret.name -}}
+{{- else -}}
+  {{- print .Release.Name "-kafka" -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Return name of secret where kafka information is stored
+*/}}
+{{- define "wandb.kafka.passwordSecret.passwordKey" -}}
+{{- if .Values.global.kafka.passwordSecret.name }}
+  {{- .Values.global.kafka.passwordSecret.passwordKey -}}
+{{- else -}}
+  KAFKA_CLIENT_PASSWORD
 {{- end -}}
+{{- end }}
 
 {{/*
 Return the kafka broker url port

diff --git a/charts/operator-wandb/templates/clickhouse.yaml b/charts/operator-wandb/templates/clickhouse.yaml
@@ -1,4 +1,5 @@
 ---
+{{- if not .Values.global.clickhouse.passwordSecret.name }}
 {{- $secretName := (include "wandb.clickhouse.passwordSecret" .)  }}
 apiVersion: v1
 kind: Secret
@@ -7,4 +8,5 @@ metadata:
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
-  CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }}
+  CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }}
+{{- end }}
diff --git a/charts/operator-wandb/templates/kafka.yaml b/charts/operator-wandb/templates/kafka.yaml
@@ -11,6 +11,7 @@ data:
   KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE: {{ include "wandb.kafka.runUpdatesShadowTopic" . | quote }}
   KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS: {{ include "wandb.kafka.runUpdatesShadowNumPartitions" . | quote}}
 ---
+{{- if not .Values.global.kafka.passwordSecret.name }}
 {{- $secretName := (include "wandb.kafka.passwordSecret" .)  }}
 apiVersion: v1
 kind: Secret
@@ -20,3 +21,4 @@ metadata:
     {{- include "wandb.commonLabels" . | nindent 4 }}
 data:
   KAFKA_CLIENT_PASSWORD: {{ include "wandb.kafka.password" . | b64enc }}
+{{- end }}
diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml
@@ -76,6 +76,9 @@ global:
     host: ""
     port: 8443
     password: "fake"
+    passwordSecret:
+      name: ""
+      passwordKey: "CLICKHOUSE_PASSWORD"
     database: "weave_trace_db"
     user: "default"
 
@@ -139,6 +142,13 @@ global:
     # the kafka dependency chart.
     user: &kafkaUser "wandb"
     password: &kafkaPassword "wandb"
+## The secret can be created using the following command:
+## "kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=controller-password=CONTROLLER_PASSWORD"
+## Replace SECRET_NAME, CLIENT_PASSWORD1, INTER_BROKER_PASSWORD, and CONTROLLER_PASSWORD with appropriate values.
+    passwordSecret:
+      name: &kafkaSecretName ""
+      # This (client-passwords) should match the key name used in the secret
+      passwordKey: "client-passwords"
     brokerHost: ""
     brokerPort: 9092
     runUpdatesShadowTopic: ""
@@ -380,6 +390,8 @@ kafka:
       users:
         - *kafkaUser
       passwords: *kafkaPassword
+    existingSecret: *kafkaSecretName
+
   kraft:
     # This field is a UUID. It is *strongly* recommended to supply a new UUID yourself for production installs.
     clusterId: "ffFF1H3AQKGdBnsqAbJKew"