ops/nixos: add totoro as prometheus box; enable node-exporter everywhere

ops/nixos/default.nix

@@ -22,7 +22,19 @@ let
   evaledSystems = mapAttrs systemFor systemCfgs;
   systemDrvs = mapAttrs (_: sys: sys.config.system.build.toplevel) evaledSystems;
   systemTailscaleIPs = lib.mapAttrs' (n: v: lib.nameValuePair v [n]) (lib.filterAttrs (n: v: v != null) (mapAttrs (_: sys: sys.config.my.ip.tailscale) evaledSystems));
+
+  systemExporters = builtins.listToAttrs (builtins.concatLists (lib.mapAttrsToList (sysName: sys: lib.mapAttrsToList (exporterName: exporter: let jobName = "${sysName}/exporters/${exporterName}"; in lib.nameValuePair jobName {
+    job_name = jobName;
+    static_configs = [{
+      targets = ["${sysName}:${toString exporter.port}"];
+      labels = {
+        system = sysName;
+        exporter = exporterName;
+      };
+    }];
+  }) (lib.filterAttrs (_: exporter: exporter.enable) sys.config.services.prometheus.exporters)) evaledSystems));
 in systemDrvs // {
   systems = systemDrvs;
+  systemExporters = systemExporters;
   tailscaleIPs = systemTailscaleIPs;
 }

ops/nixos/lib/common.nix

@@ -140,5 +140,7 @@ in
         lib.optional (config.my.home-manager.system != null) config.my.home-manager.system
       );
     });
+
+    services.prometheus.exporters.node.enable = true;
   };
 }

ops/nixos/totoro/default.nix

@@ -84,5 +84,41 @@ in {
 
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 
+  systemd.tmpfiles.rules = [
+    "L /var/lib/export - - - - /export"
+  ];
+
+  services.prometheus = {
+    enable = true;
+    stateDir = "export/monitoring/prometheus";
+    alertmanagers = [{
+      scheme = "http";
+      static_configs = [{
+        targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];
+      }];
+    }];
+    scrapeConfigs = builtins.attrValues depot.ops.nixos.systemExporters;
+
+    pushgateway.enable = true;
+
+    alertmanager = {
+      enable = true;
+      configuration = {
+        global = {};
+        route = {
+          receiver = "default-receiver";
+        };
+        receivers = [{
+          name = "default-receiver";
+          slack_configs = [{
+            send_resolved = true;
+            api_url = secrets.monitoring.alertmanager.discord.api_url;
+            channel = "#alerts";
+          }];
+        }];
+      };
+    };
+  };
+
   system.stateVersion = "20.03";
 }