roles/prometheus: try sending text emails from alertmanager

Signed-off-by: John Helmert III <[email protected]>

roles/prometheus/files/email.tmpl

@@ -0,0 +1,43 @@
+{{ define "__subject" }}
+{{ .CommonLabels.priority }} {{ .CommonLabels.instance }} {{ .CommonLabels.alertname }} {{ .Status }}
+{{ end }}
+
+{{ define "__text_alert_list" }}{{ range . }}
+Labels:
+{{ range .Labels.SortedPairs }}{{ if lt (len .Name) 5 }}  {{ .Name }}:{{ "\t" }}{{ "\t" }}{{ .Value }}{{ else }}  {{ .Name }}:{{ "\t" }}{{ .Value }}{{ end }}
+{{ end }}Annotations:
+{{ range .Annotations.SortedPairs }}{{ if lt (len .Name) 5 }}  {{ .Name }}:{{ "\t" }}{{ "\t" }}{{ .Value }}{{ else }}  {{ .Name }}:{{ "\t" }}{{ .Value }}{{ end }}
+{{ end }}Source:
+{{ .GeneratorURL }}
+{{ end }}{{ end }}
+
+{{ define "__alert_silence_link" -}}
+    {{ .ExternalURL }}/#/silences/new?filter=%7B
+    {{- range .CommonLabels.SortedPairs -}}
+        {{- if ne .Name "alertname" -}}
+            {{- .Name }}%3D"{{- .Value -}}"%2C%20
+        {{- end -}}
+    {{- end -}}
+    alertname%3D"{{ .CommonLabels.alertname }}"%7D
+{{- end }}
+
+
+
+{{ define "email.comfault.subject" }}{{ template "__subject" . }}{{ end }}
+
+{{ define "email.example.text" }}
+{{ if gt (len .Alerts.Firing) 0 -}}
+# Alerts firing
+
+{{ template "__text_alert_list" .Alerts.Firing }}
+{{- end }}
+
+{{ if gt (len .Alerts.Resolved) 0 -}}
+# Alerts resolved
+
+{{ template "__text_alert_list" .Alerts.Resolved }}
+{{- end }}
+
+Manage notification:
+{{ template "__alert_silence_link" . }}
+{{- end }}

roles/prometheus/tasks/prometheus.yml

@@ -1,5 +1,5 @@
 ---
-- name: Create prometheus config directories
+- name: Create prometheus and alertmanager config directories
   ansible.builtin.file:
     path: "{{ item }}"
     state: directory
@@ -10,7 +10,11 @@
     - "{{ prom_confdir }}"
     - "{{ prom_confdir }}/alerts"
     - "{{ am_confdir }}"
-    - "{{ am_confdir }}/template"
+    - "{{ am_confdir }}/templates"
+- name: Remove old templates directory
+  ansible.builtin.file:
+    path: "{{ am_confdir }}/template"
+    state: absent
 - name: Create prometheus data directory
   ansible.builtin.file:
     path: "{{ prom_datadir }}"
@@ -47,6 +51,14 @@
     group: root
     mode: "0644"
     # validate: amtool check-config %s
+- name: Install alertmanager email template
+  notify: Restart alertmanager
+  ansible.builtin.copy:
+    src: email.tmpl
+    dest: "{{ am_confdir }}/templates/email.tmpl"
+    owner: root
+    group: root
+    mode: "0644"
 - name: Install docker-compose.yml
   ansible.builtin.template:
     src: docker-compose.yml.j2

roles/prometheus/templates/am_config.yml.j2

@@ -6,6 +6,7 @@ global:
 
 # The directory from which notification templates are read.
 templates:
+  - '/etc/alertmanager/templates/email.tmpl'
 
 # The root route on which each incoming alert enters.
 route:
@@ -100,6 +101,7 @@ receivers:
   - name: 'team-X-mails'
     email_configs:
       - to: '{{ my_email }}'
+        text: '{% raw %}{{ template "email.example.text" }}{% endraw %}'
 
   - name: 'team-X-pager'
     email_configs: