totoro: add NodeExporterDown alerts

2021-03-15 01:25:19 +00:00 · 2021-03-15 01:25:19 +00:00 · 04a7792ad1
commit 04a7792ad1
parent dc2fb8e2b1
1 changed files with 10 additions and 0 deletions
--- a/ops/nixos/totoro/default.nix
+++ b/ops/nixos/totoro/default.nix
@ -247,6 +247,16 @@ in {
            annotations:
              summary: "Blade: Single Blade Power Usage Out of Bounds"
              description: "{{ $labels.system }} has power usage of {{ $value }}, which is out of expected bounds."
+      - name: availability
+        rules:
+        - alert: NodeExporterDown
+          expr: up{exporter="node", system=~"(blade-(tuvok|paris|janeway|torres)|kusakabe|marukuru|swann|totoro|clouvider-.*|etheroute-.*)"} < 1
+          for: 30m
+          labels:
+            severity: page
+          annotations:
+            summary: "Node exporter no longer scrapable"
+            description: "{{ $labels.system }} is not reachable from totoro."
      ''
    ];