From b51cf0628266604a1a5c326638a85f814f1a99c8 Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Sun, 14 Mar 2021 01:13:37 +0000
Subject: [PATCH] totoro: prometheus: remove valveindexinstock, add snmp

---
 ops/nixos/default.nix        |  2 +-
 ops/nixos/totoro/default.nix | 57 ++++++++++++++++--------------------
 2 files changed, 27 insertions(+), 32 deletions(-)

diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix
index ff2703c494..02de3ed291 100644
--- a/ops/nixos/default.nix
+++ b/ops/nixos/default.nix
@@ -51,7 +51,7 @@ let
           };
         }];
       };
-      stockExporters = lib.mapAttrsToList (exporterName: exporter: mkExporter exporterName exporter.port) (lib.filterAttrs (_: exporter: exporter.enable) sys.config.services.prometheus.exporters);
+      stockExporters = lib.mapAttrsToList (exporterName: exporter: mkExporter exporterName exporter.port) (lib.filterAttrs (exporterName: exporter: exporterName != "snmp" && exporter.enable) sys.config.services.prometheus.exporters);
       customExporters = lib.mapAttrsToList mkExporter sys.config.my.prometheus.additionalExporterPorts;
     in
       stockExporters ++ customExporters;
diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix
index 8742d9bbd8..e247e9053c 100644
--- a/ops/nixos/totoro/default.nix
+++ b/ops/nixos/totoro/default.nix
@@ -196,25 +196,30 @@ in {
         targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];
       }];
     }];
-    scrapeConfigs = builtins.attrValues depot.ops.nixos.systemExporters;
+    scrapeConfigs = (builtins.attrValues depot.ops.nixos.systemExporters) ++ [{
+      job_name = "blade-oa/snmp";
+      metrics_path = "/snmp";
+      params = {
+        module = ["hpe"];
+      };
+      static_configs = [{
+        targets = ["10.100.1.200"];
+      }];
+      relabel_configs = [{
+        source_labels = ["__address__"];
+        target_label = "__param_target";
+      } {
+        source_labels = ["__param_target"];
+        target_label = "instance";
+      } {
+        target_label = "__address__";
+        replacement = "totoro:${toString config.services.prometheus.exporters.snmp.port}";
+      }];
+    }];
 
     pushgateway.enable = true;
 
-    rules = [
-      ''
-        groups:
-        - name: valve_index
-          rules:
-          - alert: ValveIndexInStock
-            expr: valve_index_in_stock > 0
-            for: 1s
-            labels:
-              severity: page
-            annotations:
-              summary: "Valve Index in Stock!"
-              description: "The Valve Index is back in stock."
-      ''
-    ];
+    rules = [];
 
     alertmanager = {
       enable = true;
@@ -231,6 +236,11 @@ in {
         }];
       };
     };
+
+    exporters.snmp = {
+      enable = true;
+      configurationPath = depot.nix.pkgs.prometheus-snmp-config;
+    };
   };
   services.grafana = {
     enable = true;
@@ -279,21 +289,6 @@ in {
     };
   };
 
-  systemd.services.valveindexinstock = {
-    enable = true;
-    serviceConfig = {
-      ExecStart = "${depot.py.valveindexinstock}/bin/valveindexinstock";
-      DynamicUser = true;
-      MountAPIVFS = true;
-      PrivateTmp = true;
-      PrivateUsers = true;
-      ProtectControlGroups = true;
-      ProtectKernelModules = true;
-      ProtectKernelTunables = true;
-    };
-  };
-  my.prometheus.additionalExporterPorts.valveindexinstock = 9998;
-
   systemd.services.alertmanager-discord = {
     enable = true;
     serviceConfig = {