Merge branch 'main' into otel

charts/operator-wandb/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: operator-wandb
 description: A Helm chart for deploying W&B to Kubernetes
 type: application
-version: 0.10.13
+version: 0.10.21
 appVersion: 1.0.0
 icon: https://wandb.ai/logo.svg
 

charts/operator-wandb/charts/app/templates/deployment.yaml

@@ -34,6 +34,7 @@ spec:
         {{-   toYaml .Values.pod.annotations | nindent 4 }}
         {{- end }}
     spec:
+      serviceAccountName: {{ include "app.serviceAccountName" . }}
       {{- if .tolerations }}
       tolerations:
         {{- toYaml .tolerations | nindent 8 }}
@@ -100,6 +101,13 @@ spec:
             - name: MYSQL
               value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_HOST):$(MYSQL_PORT)/$(MYSQL_DATABASE)"
 
+            - name: WEAVE_SERVICE
+              value: "{{ .Release.Name }}-weave:9994"
+            - name: PARQUET_HOST
+              value: "http://{{ .Release.Name }}-parquet:8087"
+            - name: PARQUET_ENABLED
+              value: "true"
+
             {{- if ne (include "wandb.redis.password" .) "" }}
             - name: REDIS_PASSWORD
               valueFrom:
@@ -114,14 +122,6 @@ spec:
             - name: REDIS
               value: "{{ include "app.redis" . | trim }}"
 
-            - name: WEAVE_SERVICE
-              value: "{{ .Release.Name }}-weave:9994"
-
-            - name: PARQUET_HOST
-              value: "http://{{ .Release.Name }}-parquet:8087"
-            - name: PARQUET_ENABLED
-              value: "true"
-
             - name: SLACK_CLIENT_ID
               value: {{ .Values.global.slack.clientId }}
             - name: SLACK_SECRET
@@ -175,6 +175,11 @@ spec:
                   name: "{{ include "wandb.bucket.secret" . }}"
                   key: ACCESS_KEY
                   optional: true
+
+            - name: GORILLA_CUSTOMER_SECRET_STORE_K8S_CONFIG_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
 
             {{- include "app.extraEnv" (dict "global" $.Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}

charts/operator-wandb/charts/app/templates/role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "app.fullname" . }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.labels" . | nindent 4 }}
+    {{- if .Values.role.labels -}}
+    {{-   toYaml .Values.role.labels | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- if .Values.role.annotations -}}
+    {{-   toYaml .Values.role.annotations | nindent 4 }}
+    {{- end }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get"]

charts/operator-wandb/charts/app/templates/rolebinding.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "app.fullname" . }}
+  labels:
+    {{- include "wandb.commonLabels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.labels" . | nindent 4 }}
+    {{- if .Values.role.labels -}}
+    {{-   toYaml .Values.role.labels | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- if .Values.role.annotations -}}
+    {{-   toYaml .Values.role.annotations | nindent 4 }}
+    {{- end }}
+roleRef:
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+  name: {{ include "app.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "app.fullname" . }}
+    namespace: {{ .Release.Namespace }}

charts/operator-wandb/old/prometheus/templates/serviceaccount.yaml → charts/operator-wandb/charts/app/templates/serviceaccount.yaml

@@ -1,12 +1,12 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ include "prometheus.fullname" . }}
+  name: {{ include "app.serviceAccountName" . }}
   namespace: {{ $.Release.Namespace }}
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
-    {{- include "prometheus.commonLabels" . | nindent 4 }}
-    {{- include "prometheus.labels" . | nindent 4 }}
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.labels" . | nindent 4 }}
     {{- if .Values.serviceAccount.labels -}}
     {{-   toYaml .Values.serviceAccount.labels | nindent 4 }}
     {{- end }}

charts/operator-wandb/charts/app/values.yaml

@@ -49,3 +49,9 @@ resources:
   limits:
     cpu: 4000m
     memory: 8Gi
+
+serviceAccount:
+  create: true
+
+role: {}
+roleBinding: {}

charts/operator-wandb/charts/parquet/templates/_helpers.tpl

@@ -25,6 +25,16 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
+{{- define "parquet.redis" -}}
+{{- $cs := include "wandb.redis.connectionString" . }}
+{{- $ca := include "wandb.redis.caCert" . }}
+{{- if $ca }}
+{{- printf "%s?tls=true&caCertPath=/etc/ssl/certs/redis_ca.pem&ttlInSeconds=604800" $cs -}}
+{{- else }}
+{{- print $cs -}}
+{{- end }}
+{{- end }}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}
@@ -108,6 +118,10 @@ app deployments.
 {{- printf "gs://%s" .Values.global.bucket.name -}}
 {{- end }}
 {{- if eq .Values.global.bucket.provider "s3" }}
+{{- if and .Values.global.bucket.accessKey .Values.global.bucket.secretKey}}
+{{- printf "s3://%s:%s@%s" .Values.global.bucket.accessKey .Values.global.bucket.secretKey .Values.global.bucket.name -}}
+{{- else }}
 {{- printf "s3://%s" .Values.global.bucket.name -}}
 {{- end }}
 {{- end }}
+{{- end }}

charts/operator-wandb/charts/parquet/templates/deployment.yaml

@@ -44,6 +44,12 @@ spec:
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          volumeMounts:
+            {{- if ne (include "wandb.redis.caCert" .) "" }}
+            - name: {{ include "parquet.fullname" . }}-redis-ca
+              mountPath: /etc/ssl/certs/redis_ca.pem
+              subPath: redis_ca.pem
+            {{- end }}
           ports:
             - name: parquet
               containerPort: 8087
@@ -71,6 +77,23 @@ spec:
             - name: MYSQL
               value: "mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@$(MYSQL_HOST):$(MYSQL_PORT)/$(MYSQL_DATABASE)"
 
+            {{- if ne (include "wandb.redis.password" .) "" }}
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "wandb.redis.passwordSecret" . }}
+                  key: REDIS_PASSWORD
+            {{- end }}
+            - name: REDIS_PORT
+              value: "{{ include "wandb.redis.port" . }}"
+            - name: REDIS_HOST
+              value: "{{ include "wandb.redis.host" . }}"
+            - name: REDIS
+              value: "{{ include "parquet.redis" . | trim }}"
+
+            - name: GORILLA_SETTINGS_CACHE
+              value: "{{ include "parquet.redis" . | trim }}"
+
             - name: BUCKET
               value: "{{ include "parquet.bucket" . }}"
             - name: BUCKET_QUEUE
@@ -79,10 +102,25 @@ spec:
               value: {{ .Values.global.bucket.region }}
             - name: AWS_S3_KMS_ID
               value: "{{ .Values.global.bucket.kmsKey }}"
+            - name: AZURE_STORAGE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ include "wandb.bucket.secret" . }}"
+                  key: ACCESS_KEY
+                  optional: true
 
             {{- include "parquet.extraEnv" (dict "global" .Values.global "local" .Values) | nindent 12 }}
             {{- include "wandb.extraEnvFrom" (dict "root" $ "local" .) | nindent 12 }}
 
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        {{- if ne (include "wandb.redis.caCert" .) "" }}
+        - name: {{ include "parquet.fullname" . }}-redis-ca
+          secret:
+            secretName: "{{ include "wandb.redis.passwordSecret" . }}"
+            items:
+              - key: REDIS_CA_CERT
+                path: redis_ca.pem
+        {{- end }}
 {{- end }}

charts/operator-wandb/charts/parquet/templates/service.yaml

@@ -1,12 +1,11 @@
-{{- if .Values.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "weave.fullname" . }}
+  name: {{ include "parquet.fullname" . }}
   labels:
     {{- include "wandb.commonLabels" . | nindent 4 }}
-    {{- include "weave.labels" . | nindent 4 }}
-    {{- include "weave.commonLabels" . | nindent 4 }}
+    {{- include "parquet.labels" . | nindent 4 }}
+    {{- include "parquet.commonLabels" . | nindent 4 }}
     {{- if .Values.service.labels -}}
     {{-   toYaml .Values.service.labels | nindent 4 }}
     {{- end }}
@@ -21,5 +20,4 @@ spec:
       protocol: TCP
       name: parquet
   selector:
-    {{- include "weave.labels" . | nindent 4 }}
-{{- end }}
+    {{- include "parquet.labels" . | nindent 4 }}