Merge pull request #87 from caos/avoid-no-data

fix: include up metric in etcd check
caos · Apr 22, 2020 · bb2f20f · bb2f20f
2 parents e3fcd0d + d7a9e47
commit bb2f20f
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 5 deletions.
diff --git a/dashboards/health/json/cluster-health.json b/dashboards/health/json/cluster-health.json
@@ -2861,7 +2861,7 @@
       "thresholds": "0.001,1",
       "timeFrom": null,
       "timeShift": null,
-      "title": "etcd $master has a leader",
+      "title": "etcd $master",
       "type": "singlestat",
       "valueFontSize": "80%",
       "valueMaps": [

diff --git a/dashboards/health/json/instance-health.json b/dashboards/health/json/instance-health.json
@@ -1022,7 +1022,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "caos_etcd_server_has_leader{ instance=\"$instance\"}",
+          "expr": "caos_etcd_server_has_leader_and_is_up{ instance=\"$instance\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1042,7 +1042,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "etcd has a leader",
+      "title": "etcd is up and has a leader",
       "tooltip": {
         "shared": true,
         "sort": 0,

diff --git a/internal/bundle/application/applications/prometheus/config/scrapeconfig.go b/internal/bundle/application/applications/prometheus/config/scrapeconfig.go
@@ -125,7 +125,7 @@ func getEtcd() *helm.AdditionalScrapeConfig {
 		SourceLabels: []string{"__name__"},
 		TargetLabel:  "__name__",
 		Regex:        "etcd_server_has_leader",
-		Replacement:  "caos_etcd_server_has_leader",
+		Replacement:  "dist_etcd_server_has_leader",
 	}}
 
 	return &helm.AdditionalScrapeConfig{

diff --git a/internal/bundle/application/applications/prometheus/helm/rules.go b/internal/bundle/application/applications/prometheus/helm/rules.go
@@ -120,7 +120,9 @@ groups:
      record: caos_node_ready
    - expr: min_over_time(caos_node_ready[5m])
      record: caos_k8s_node_ryg
-   - expr: avg_over_time(caos_etcd_server_has_leader[5m])
+   - expr: dist_etcd_server_has_leader or on(instance) up{job="caos_remote_etcd"}
+     record: caos_etcd_server_has_leader_and_is_up
+   - expr: min_over_time(caos_etcd_server_has_leader_and_is_up[5m])
      record: caos_etcd_ryg
    - expr: |-
        clamp_max(