From 20e70bd4a6514de6008fadb38df793cfbf97b048 Mon Sep 17 00:00:00 2001 From: amanpruthi Date: Wed, 27 Nov 2024 17:26:46 +0530 Subject: [PATCH 1/5] Added support for clickhouse and kafka user secret ref --- .../weave-trace/templates/deployment.yaml | 4 ++-- .../weave-trace/templates/migrate-hook.yaml | 2 +- charts/operator-wandb/templates/_clickhouse.tpl | 17 ++++++++++++++++- charts/operator-wandb/templates/_kafka.tpl | 17 ++++++++++++++++- charts/operator-wandb/templates/clickhouse.yaml | 4 +++- charts/operator-wandb/templates/kafka.yaml | 2 ++ charts/operator-wandb/values.yaml | 12 ++++++++++++ 7 files changed, 52 insertions(+), 6 deletions(-) diff --git a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml index d83fbb09..7e54edda 100644 --- a/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml +++ b/charts/operator-wandb/charts/weave-trace/templates/deployment.yaml @@ -60,7 +60,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} {{- end }} @@ -97,7 +97,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} {{- if not .Values.datadog.enabled }} diff --git a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml index 39931cdb..306e0172 100644 --- a/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml +++ b/charts/operator-wandb/charts/weave-trace/templates/migrate-hook.yaml @@ -61,7 +61,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.clickhouse.passwordSecret" . }} - key: CLICKHOUSE_PASSWORD + key: {{ include "wandb.clickhouse.passwordSecret.passwordKey" . }} {{- include "weaveTrace.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }} {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }} restartPolicy: "Never" diff --git a/charts/operator-wandb/templates/_clickhouse.tpl b/charts/operator-wandb/templates/_clickhouse.tpl index 5ceca7dd..b9e5fd1a 100644 --- a/charts/operator-wandb/templates/_clickhouse.tpl +++ b/charts/operator-wandb/templates/_clickhouse.tpl @@ -9,8 +9,23 @@ Return the kafka client password Return name of secret where clickhouse information is stored */}} {{- define "wandb.clickhouse.passwordSecret" -}} -{{- print .Release.Name "-clickhouse" -}} +{{- if .Values.global.clickhouse.passwordSecret.name }} + {{- .Values.global.clickhouse.passwordSecret.name -}} +{{- else -}} + {{- print .Release.Name "-clickhouse" -}} +{{- end -}} +{{- end }} + +{{/* +Return name of secret where clickhouse information is stored +*/}} +{{- define "wandb.clickhouse.passwordSecret.passwordKey" -}} +{{- if .Values.global.clickhouse.passwordSecret.name }} + {{- .Values.global.clickhouse.passwordSecret.passwordKey -}} +{{- else -}} + CLICKHOUSE_PASSWORD {{- end -}} +{{- end }} {{/* Return the redis host diff --git a/charts/operator-wandb/templates/_kafka.tpl b/charts/operator-wandb/templates/_kafka.tpl index d53f45e6..53318916 100644 --- a/charts/operator-wandb/templates/_kafka.tpl +++ b/charts/operator-wandb/templates/_kafka.tpl @@ -31,8 +31,23 @@ Return the kafka client password Return name of secret where kafka information is stored */}} {{- define "wandb.kafka.passwordSecret" -}} -{{- print .Release.Name "-kafka" -}} +{{- if .Values.global.kafka.passwordSecret.name }} + {{- .Values.global.kafka.passwordSecret.name -}} +{{- else -}} + {{- print .Release.Name "-kafka" -}} +{{- end -}} +{{- end }} + +{{/* +Return name of secret where kafka information is stored +*/}} +{{- define "wandb.kafka.passwordSecret.passwordKey" -}} +{{- if .Values.global.kafka.passwordSecret.name }} + {{- .Values.global.kafka.passwordSecret.passwordKey -}} +{{- else -}} + KAFKA_CLIENT_PASSWORD {{- end -}} +{{- end }} {{/* Return the kafka broker url port diff --git a/charts/operator-wandb/templates/clickhouse.yaml b/charts/operator-wandb/templates/clickhouse.yaml index d4f4a391..5824c611 100644 --- a/charts/operator-wandb/templates/clickhouse.yaml +++ b/charts/operator-wandb/templates/clickhouse.yaml @@ -1,4 +1,5 @@ --- +{{- if not .Values.global.clickhouse.passwordSecret.name }} {{- $secretName := (include "wandb.clickhouse.passwordSecret" .) }} apiVersion: v1 kind: Secret @@ -7,4 +8,5 @@ metadata: labels: {{- include "wandb.commonLabels" . | nindent 4 }} data: - CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }} \ No newline at end of file + CLICKHOUSE_PASSWORD: {{ include "wandb.clickhouse.password" . | b64enc }} +{{- end }} diff --git a/charts/operator-wandb/templates/kafka.yaml b/charts/operator-wandb/templates/kafka.yaml index ac074f87..a1eea1a9 100644 --- a/charts/operator-wandb/templates/kafka.yaml +++ b/charts/operator-wandb/templates/kafka.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.global.kafka.passwordSecret.name }} {{- $secretName := (include "wandb.kafka.passwordSecret" .) }} apiVersion: v1 kind: Secret @@ -7,3 +8,4 @@ metadata: {{- include "wandb.commonLabels" . | nindent 4 }} data: KAFKA_CLIENT_PASSWORD: {{ include "wandb.kafka.password" . | b64enc }} +{{- end }} diff --git a/charts/operator-wandb/values.yaml b/charts/operator-wandb/values.yaml index a118a324..03e057ff 100644 --- a/charts/operator-wandb/values.yaml +++ b/charts/operator-wandb/values.yaml @@ -76,6 +76,9 @@ global: host: "" port: 8443 password: "fake" + passwordSecret: + name: "" + passwordKey: "CLICKHOUSE_PASSWORD" database: "weave_trace_db" user: "default" @@ -141,6 +144,13 @@ global: # the kafka dependency chart. user: &kafkaUser "wandb" password: &kafkaPassword "wandb" +## The secret can be created using the following command: +## "kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=controller-password=CONTROLLER_PASSWORD" +## Replace SECRET_NAME, CLIENT_PASSWORD1, INTER_BROKER_PASSWORD, and CONTROLLER_PASSWORD with appropriate values. + passwordSecret: + name: &kafkaSecretName "" + # This (client-passwords) should match the key name used in the secret + passwordKey: "client-passwords" brokerHost: "" brokerPort: 9092 runUpdatesShadowTopic: "" @@ -372,6 +382,8 @@ kafka: users: - *kafkaUser passwords: *kafkaPassword + existingSecret: *kafkaSecretName + kraft: # This field is a UUID. It is *strongly* recommended to supply a new UUID yourself for production installs. clusterId: "ffFF1H3AQKGdBnsqAbJKew" From 8a19a94e7ec1aee8c8f16ba81bd17b0dd7310149 Mon Sep 17 00:00:00 2001 From: velotioaastha Date: Thu, 5 Dec 2024 19:05:33 +0530 Subject: [PATCH 2/5] resolve conflicts --- charts/operator-wandb/Chart.lock | 2 +- charts/operator-wandb/Chart.yaml | 2 +- charts/operator-wandb/charts/app/templates/deployment.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock index 5fe45c51..a80f94a0 100644 --- a/charts/operator-wandb/Chart.lock +++ b/charts/operator-wandb/Chart.lock @@ -51,4 +51,4 @@ dependencies: repository: file://../wandb-base version: 0.1.1 digest: sha256:af1ce94a6c5f99cc9d9df3d10db200ab45b976fd24bd65f85d37e5a32517c5b8 -generated: "2024-12-02T09:48:06.166674-06:00" +generated: "2024-12-05T19:04:56.180352+05:30" diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml index 85381df5..0a40b670 100644 --- a/charts/operator-wandb/Chart.yaml +++ b/charts/operator-wandb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: operator-wandb description: A Helm chart for deploying W&B to Kubernetes type: application -version: 0.20.0 +version: 0.20.3 appVersion: 1.0.0 icon: https://wandb.ai/logo.svg diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml index 230e2ec4..c9ba05da 100644 --- a/charts/operator-wandb/charts/app/templates/deployment.yaml +++ b/charts/operator-wandb/charts/app/templates/deployment.yaml @@ -245,7 +245,7 @@ spec: valueFrom: secretKeyRef: name: {{ include "wandb.kafka.passwordSecret" . }} - key: KAFKA_CLIENT_PASSWORD + key: {{ include "wandb.kafka.passwordSecret.passwordKey" .}} - name: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE value: {{ include "wandb.kafka.runUpdatesShadowTopic" .}} - name: KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS From 0cecf90d14f27ae9a5bc1f8417e0ca7aa840e754 Mon Sep 17 00:00:00 2001 From: velotioaastha Date: Thu, 19 Dec 2024 10:19:34 +0530 Subject: [PATCH 3/5] updated chart version --- charts/operator-wandb/Chart.lock | 2 +- charts/operator-wandb/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock index a80f94a0..0c6c4723 100644 --- a/charts/operator-wandb/Chart.lock +++ b/charts/operator-wandb/Chart.lock @@ -51,4 +51,4 @@ dependencies: repository: file://../wandb-base version: 0.1.1 digest: sha256:af1ce94a6c5f99cc9d9df3d10db200ab45b976fd24bd65f85d37e5a32517c5b8 -generated: "2024-12-05T19:04:56.180352+05:30" +generated: "2024-12-19T10:19:07.523752+05:30" diff --git a/charts/operator-wandb/Chart.yaml b/charts/operator-wandb/Chart.yaml index 0a40b670..552d6172 100644 --- a/charts/operator-wandb/Chart.yaml +++ b/charts/operator-wandb/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: operator-wandb description: A Helm chart for deploying W&B to Kubernetes type: application -version: 0.20.3 +version: 0.21.9 appVersion: 1.0.0 icon: https://wandb.ai/logo.svg From 027147b19a93a450809daa12f673e2dd49a244ab Mon Sep 17 00:00:00 2001 From: velotioaastha Date: Thu, 19 Dec 2024 10:25:37 +0530 Subject: [PATCH 4/5] resolv conflicts --- .../charts/app/templates/deployment.yaml | 1 + charts/operator-wandb/templates/kafka.yaml | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/charts/operator-wandb/charts/app/templates/deployment.yaml b/charts/operator-wandb/charts/app/templates/deployment.yaml index 8c2d13ca..9dffe105 100644 --- a/charts/operator-wandb/charts/app/templates/deployment.yaml +++ b/charts/operator-wandb/charts/app/templates/deployment.yaml @@ -246,6 +246,7 @@ spec: secretKeyRef: name: {{ include "wandb.kafka.passwordSecret" . }} key: {{ include "wandb.kafka.passwordSecret.passwordKey" .}} + optional: true - name: KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE value: {{ include "wandb.kafka.runUpdatesShadowTopic" .}} - name: KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS diff --git a/charts/operator-wandb/templates/kafka.yaml b/charts/operator-wandb/templates/kafka.yaml index a1eea1a9..35c9b24d 100644 --- a/charts/operator-wandb/templates/kafka.yaml +++ b/charts/operator-wandb/templates/kafka.yaml @@ -1,3 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-kafka-configmap + labels: + {{- include "wandb.commonLabels" . | nindent 4 }} +data: + KAFKA_BROKER_HOST: "{{ include "wandb.kafka.brokerHost" . }}" + KAFKA_BROKER_PORT: "{{ include "wandb.kafka.brokerPort" . }}" + KAFKA_CLIENT_USER: "{{ include "wandb.kafka.user" . }}" + KAFKA_TOPIC_RUN_UPDATE_SHADOW_QUEUE: {{ include "wandb.kafka.runUpdatesShadowTopic" . | quote }} + KAFKA_RUN_UPDATE_SHADOW_QUEUE_NUM_PARTITIONS: {{ include "wandb.kafka.runUpdatesShadowNumPartitions" . | quote}} +--- {{- if not .Values.global.kafka.passwordSecret.name }} {{- $secretName := (include "wandb.kafka.passwordSecret" .) }} apiVersion: v1 From 7fad453f779f895a2058777267275fcb3fdbaaad Mon Sep 17 00:00:00 2001 From: velotioaastha Date: Thu, 19 Dec 2024 11:01:30 +0530 Subject: [PATCH 5/5] helm lint --- charts/operator-wandb/Chart.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/operator-wandb/Chart.lock b/charts/operator-wandb/Chart.lock index 08d1fa9f..b2941e87 100644 --- a/charts/operator-wandb/Chart.lock +++ b/charts/operator-wandb/Chart.lock @@ -31,7 +31,7 @@ dependencies: version: 25.3.5 - name: etcd repository: oci://registry-1.docker.io/bitnamicharts - version: 10.6.1 + version: 10.7.1 - name: bufstream repository: file://charts/bufstream version: 0.3.1 @@ -56,5 +56,5 @@ dependencies: - name: wandb-base repository: file://../wandb-base version: 0.1.1 -digest: sha256:af1ce94a6c5f99cc9d9df3d10db200ab45b976fd24bd65f85d37e5a32517c5b8 -generated: "2024-12-19T10:19:07.523752+05:30" +digest: sha256:3c02029e1921428d7f45866291a8c42b79894505ee47864c54f34fcd847b2793 +generated: "2024-12-19T11:00:55.672581+05:30"