diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix index 2493ea7bf3..885f1a1468 100644 --- a/ops/nixos/default.nix +++ b/ops/nixos/default.nix @@ -22,7 +22,19 @@ let evaledSystems = mapAttrs systemFor systemCfgs; systemDrvs = mapAttrs (_: sys: sys.config.system.build.toplevel) evaledSystems; systemTailscaleIPs = lib.mapAttrs' (n: v: lib.nameValuePair v [n]) (lib.filterAttrs (n: v: v != null) (mapAttrs (_: sys: sys.config.my.ip.tailscale) evaledSystems)); + + systemExporters = builtins.listToAttrs (builtins.concatLists (lib.mapAttrsToList (sysName: sys: lib.mapAttrsToList (exporterName: exporter: let jobName = "${sysName}/exporters/${exporterName}"; in lib.nameValuePair jobName { + job_name = jobName; + static_configs = [{ + targets = ["${sysName}:${toString exporter.port}"]; + labels = { + system = sysName; + exporter = exporterName; + }; + }]; + }) (lib.filterAttrs (_: exporter: exporter.enable) sys.config.services.prometheus.exporters)) evaledSystems)); in systemDrvs // { systems = systemDrvs; + systemExporters = systemExporters; tailscaleIPs = systemTailscaleIPs; } diff --git a/ops/nixos/lib/common.nix b/ops/nixos/lib/common.nix index 1375c86530..096258b0ed 100644 --- a/ops/nixos/lib/common.nix +++ b/ops/nixos/lib/common.nix @@ -140,5 +140,7 @@ in lib.optional (config.my.home-manager.system != null) config.my.home-manager.system ); }); + + services.prometheus.exporters.node.enable = true; }; } diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix index 0cc2538297..4bce738581 100644 --- a/ops/nixos/totoro/default.nix +++ b/ops/nixos/totoro/default.nix @@ -84,5 +84,41 @@ in { networking.firewall.allowedTCPPorts = [ 80 443 ]; + systemd.tmpfiles.rules = [ + "L /var/lib/export - - - - /export" + ]; + + services.prometheus = { + enable = true; + stateDir = "export/monitoring/prometheus"; + alertmanagers = [{ + scheme = "http"; + static_configs = [{ + targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"]; + }]; + }]; + scrapeConfigs = builtins.attrValues depot.ops.nixos.systemExporters; + + pushgateway.enable = true; + + alertmanager = { + enable = true; + configuration = { + global = {}; + route = { + receiver = "default-receiver"; + }; + receivers = [{ + name = "default-receiver"; + slack_configs = [{ + send_resolved = true; + api_url = secrets.monitoring.alertmanager.discord.api_url; + channel = "#alerts"; + }]; + }]; + }; + }; + }; + system.stateVersion = "20.03"; }