From 04a7792ad17197d58c67fac645c64130b3f3070e Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Mon, 15 Mar 2021 01:25:19 +0000
Subject: [PATCH] totoro: add NodeExporterDown alerts

---
 ops/nixos/totoro/default.nix | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix
index eb56951ddb..ad784e9e5f 100644
--- a/ops/nixos/totoro/default.nix
+++ b/ops/nixos/totoro/default.nix
@@ -247,6 +247,16 @@ in {
             annotations:
               summary: "Blade: Single Blade Power Usage Out of Bounds"
               description: "{{ $labels.system }} has power usage of {{ $value }}, which is out of expected bounds."
+      - name: availability
+        rules:
+        - alert: NodeExporterDown
+          expr: up{exporter="node", system=~"(blade-(tuvok|paris|janeway|torres)|kusakabe|marukuru|swann|totoro|clouvider-.*|etheroute-.*)"} < 1
+          for: 30m
+          labels:
+            severity: page
+          annotations:
+            summary: "Node exporter no longer scrapable"
+            description: "{{ $labels.system }} is not reachable from totoro."
       ''
     ];