From 9177b04eb7941235238a57ba2281c0e7f63aa009 Mon Sep 17 00:00:00 2001 From: Aman Pruthi Date: Thu, 19 Dec 2024 15:39:20 +0530 Subject: [PATCH] Added support ClickHouse and Kafka passwords as secret key variables (#267) Co-authored-by: velotioaastha Co-authored-by: Ish Shah --- charts/operator-wandb/Chart.lock | 6 +++--- charts/operator-wandb/Chart.yaml | 2 +- .../charts/app/templates/deployment.yaml | 6 +++++- .../weave-trace/templates/deployment.yaml | 4 ++-- .../weave-trace/templates/migrate-hook.yaml | 2 +- charts/operator-wandb/templates/_clickhouse.tpl | 17 ++++++++++++++++- charts/operator-wandb/templates/_kafka.tpl | 17 ++++++++++++++++- charts/operator-wandb/templates/clickhouse.yaml | 4 +++- charts/operator-wandb/templates/kafka.yaml | 2 ++ charts/operator-wandb/values.yaml | 12 ++++++++++++ 10 files changed, 61 insertions(+), 11 deletions(-) diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock index def9b848..b2941e87 100644 --- a/charts/operator-wandb/Chart.lock +++ b/charts/operator-wandb/Chart.lock @@ -31,7 +31,7 @@ dependencies: version: 25.3.5 - name: etcd repository: oci://registry-1.docker.io/bitnamicharts - version: 10.6.1 + version: 10.7.1 - name: bufstream repository: file://charts/bufstream version: 0.3.1 @@ -56,5 +56,5 @@ dependencies: - name: wandb-base repository: file://../wandb-base version: 0.1.1 -digest: sha256:1eb284a2d6588f3e3d90293abb99f0e25b02fd62a87758f023d809879a82e31d -generated: "2024-12-06T11:08:12.69179-08:00" +digest: sha256:3c02029e1921428d7f45866291a8c42b79894505ee47864c54f34fcd847b2793 +generated: "2024-12-19T11:00:55.672581+05:30" diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml index 13006d9e..48a0dcc0 100644 --- a/charts/operator-wandb/Chart.yaml +++ b/charts/operator-wandb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: operator-wandb description: A Helm chart for deploying W&B to Kubernetes type: application -version: 0.21.8 +version: 0.21.9 appVersion: 1.0.0 icon: https://wandb.ai/logo.svg diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml index bcc074b1..e298b6d8 100644 --- a/charts/operator-wandb/charts/app/templates/deployment.yaml +++ b/charts/operator-wandb/charts/app/templates/deployment.yaml @@ -257,8 +257,12 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.kafka.passwordSecret" . }} - key: KAFKA_CLIENT_PASSWORD + key: {{ include "wandb.kafka.passwordSecret.passwordKey" .}} optional: true + - name: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE + value: {{ include "wandb.kafka.runUpdatesShadowTopic" .}} + - name: KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS + value: "{{ include "wandb.kafka.runUpdatesShadowNumPartitions" .}}" {{- end }} - name: GORILLA_RUN_UPDATE_SHADOW_QUEUE value: > diff --git a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml index 3bb3f854..e65ebf04 100644 --- a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml +++ b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml @@ -60,7 +60,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} {{- end }} @@ -97,7 +97,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} {{- if not .Values.datadog.enabled }} diff --git a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml index 39931cdb..306e0172 100644 --- a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml +++ b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml @@ -61,7 +61,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} restartPolicy: "Never" diff --git a/charts/operator-wandb/templates/_clickhouse.tpl b/charts/operator-wandb/templates/_clickhouse.tpl index 5ceca7dd..b9e5fd1a 100644 --- a/charts/operator-wandb/templates/_clickhouse.tpl +++ b/charts/operator-wandb/templates/_clickhouse.tpl @@ -9,8 +9,23 @@ Return the kafka client password Return name of secret where clickhouse information is stored */}} {{- define "wandb.clickhouse.passwordSecret" -}} -{{- print .Release.Name "-clickhouse" -}} +{{- if .Values.global.clickhouse.passwordSecret.name }} + {{- .Values.global.clickhouse.passwordSecret.name -}} +{{- else -}} + {{- print .Release.Name "-clickhouse" -}} +{{- end -}} +{{- end }} + +{{/* +Return name of secret where clickhouse information is stored +*/}} +{{- define "wandb.clickhouse.passwordSecret.passwordKey" -}} +{{- if .Values.global.clickhouse.passwordSecret.name }} + {{- .Values.global.clickhouse.passwordSecret.passwordKey -}} +{{- else -}} + CLICKHOUSE_PASSWORD {{- end -}} +{{- end }} {{/* Return the redis host diff --git a/charts/operator-wandb/templates/_kafka.tpl b/charts/operator-wandb/templates/_kafka.tpl index f3beaf6d..4d09c5d7 100644 --- a/charts/operator-wandb/templates/_kafka.tpl +++ b/charts/operator-wandb/templates/_kafka.tpl @@ -35,8 +35,23 @@ Return the kafka client password Return name of secret where kafka information is stored */}} {{- define "wandb.kafka.passwordSecret" -}} -{{- print .Release.Name "-kafka" -}} +{{- if .Values.global.kafka.passwordSecret.name }} + {{- .Values.global.kafka.passwordSecret.name -}} +{{- else -}} + {{- print .Release.Name "-kafka" -}} +{{- end -}} +{{- end }} + +{{/* +Return name of secret where kafka information is stored +*/}} +{{- define "wandb.kafka.passwordSecret.passwordKey" -}} +{{- if .Values.global.kafka.passwordSecret.name }} + {{- .Values.global.kafka.passwordSecret.passwordKey -}} +{{- else -}} + KAFKA_CLIENT_PASSWORD {{- end -}} +{{- end }} {{/* Return the kafka broker url port diff --git a/charts/operator-wandb/templates/clickhouse.yaml b/charts/operator-wandb/templates/clickhouse.yaml index d4f4a391..5824c611 100644 --- a/charts/operator-wandb/templates/clickhouse.yaml +++ b/charts/operator-wandb/templates/clickhouse.yaml @@ -1,4 +1,5 @@ --- +{{- if not .Values.global.clickhouse.passwordSecret.name }} {{- $secretName := (include "wandb.clickhouse.passwordSecret" .) }} apiVersion: v1 kind: Secret @@ -7,4 +8,5 @@ metadata: labels: {{- include "wandb.commonLabels" . | nindent 4 }} data: - CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }} \ No newline at end of file + CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }} +{{- end }} diff --git a/charts/operator-wandb/templates/kafka.yaml b/charts/operator-wandb/templates/kafka.yaml index 7f553501..35c9b24d 100644 --- a/charts/operator-wandb/templates/kafka.yaml +++ b/charts/operator-wandb/templates/kafka.yaml @@ -11,6 +11,7 @@ data: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE: {{ include "wandb.kafka.runUpdatesShadowTopic" . | quote }} KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS: {{ include "wandb.kafka.runUpdatesShadowNumPartitions" . | quote}} --- +{{- if not .Values.global.kafka.passwordSecret.name }} {{- $secretName := (include "wandb.kafka.passwordSecret" .) }} apiVersion: v1 kind: Secret @@ -20,3 +21,4 @@ metadata: {{- include "wandb.commonLabels" . | nindent 4 }} data: KAFKA_CLIENT_PASSWORD: {{ include "wandb.kafka.password" . | b64enc }} +{{- end }} diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml index e82e09e6..bffd0908 100644 --- a/charts/operator-wandb/values.yaml +++ b/charts/operator-wandb/values.yaml @@ -76,6 +76,9 @@ global: host: "" port: 8443 password: "fake" + passwordSecret: + name: "" + passwordKey: "CLICKHOUSE_PASSWORD" database: "weave_trace_db" user: "default" @@ -139,6 +142,13 @@ global: # the kafka dependency chart. user: &kafkaUser "wandb" password: &kafkaPassword "wandb" +## The secret can be created using the following command: +## "kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=controller-password=CONTROLLER_PASSWORD" +## Replace SECRET_NAME, CLIENT_PASSWORD1, INTER_BROKER_PASSWORD, and CONTROLLER_PASSWORD with appropriate values. + passwordSecret: + name: &kafkaSecretName "" + # This (client-passwords) should match the key name used in the secret + passwordKey: "client-passwords" brokerHost: "" brokerPort: 9092 runUpdatesShadowTopic: "" @@ -380,6 +390,8 @@ kafka: users: - *kafkaUser passwords: *kafkaPassword + existingSecret: *kafkaSecretName + kraft: # This field is a UUID. It is *strongly* recommended to supply a new UUID yourself for production installs. clusterId: "ffFF1H3AQKGdBnsqAbJKew"