#47: adding Prometheus for monitoring

Signed-off-by: Eugene Shishkin <[email protected]>

etc/k8s/comet-watcher/templates/comet-watcher-app.yml

@@ -25,6 +25,10 @@ spec:
       app: comet-watcher-service
   template:
     metadata:
+      annotations:
+        prometheus/scrape_enabled: "true"
+        prometheus/scrape_endpoint: "/metrics"
+        prometheus/scrape_port: "8080"
       labels:
         app: comet-watcher-service
     spec:

etc/k8s/comet-watcher/templates/prometheus/prometheus-account.yaml

@@ -0,0 +1,34 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-service-account
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-service-account
+    namespace: default

etc/k8s/comet-watcher/templates/prometheus/prometheus-config.yaml

@@ -0,0 +1,46 @@
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+data:
+  prometheus.yml: |-
+    global:
+      scrape_interval:     15s
+      evaluation_interval: 15s
+
+    rule_files:
+
+    scrape_configs:
+      - job_name: data-collector
+        kubernetes_sd_configs:
+        - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_scrape_enabled]
+            action: keep
+            regex: true
+
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            action: replace
+            target_label: application
+
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+
+          - source_labels: [__meta_kubernetes_namespace]
+            action: replace
+            target_label: kubernetes_namespace
+
+          - source_labels: [__meta_kubernetes_pod_name]
+            action: replace
+            target_label: kubernetes_pod_name
+
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_scrape_endpoint]
+            action: replace
+            target_label: __metrics_path__
+
+          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_scrape_port]
+            action: replace
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+            target_label: __address__

etc/k8s/comet-watcher/templates/prometheus/prometheus.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-service
+spec:
+  type: NodePort
+  ports:
+    - port: 9090
+      targetPort: 9090
+  selector:
+    app: prometheus-service
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-service
+  labels:
+    app: prometheus-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus-service
+  template:
+    metadata:
+      labels:
+        app: prometheus-service
+    spec:
+      serviceAccountName: prometheus-service-account
+      volumes:
+        - name: prometheus-config-storage
+          configMap:
+            name: prometheus-config
+      containers:
+        - name: prometheus
+          image: prom/prometheus:v2.20.1
+          volumeMounts:
+            - name: prometheus-config-storage
+              mountPath: /etc/prometheus
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090