From 336b75676baef250a28eba6da1c1fc9b776fcd6a Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Wed, 27 Mar 2024 00:37:57 +0000 Subject: [PATCH] ops/nixos: -blade --- ops/nixos/blade-chakotay/default.nix | 32 -- ops/nixos/blade-janeway/default.nix | 36 --- ops/nixos/blade-kim/default.nix | 32 -- ops/nixos/blade-paris/default.nix | 56 ---- ops/nixos/blade-torres/default.nix | 42 --- ops/nixos/blade-tuvok/default.nix | 272 ----------------- ops/nixos/default.nix | 6 - ops/nixos/lib/bgp.nix | 28 +- ops/nixos/lib/blade-router.nix | 417 --------------------------- ops/nixos/lib/blade.nix | 236 --------------- ops/nixos/totoro/default.nix | 45 +-- 11 files changed, 15 insertions(+), 1187 deletions(-) delete mode 100644 ops/nixos/blade-chakotay/default.nix delete mode 100644 ops/nixos/blade-janeway/default.nix delete mode 100644 ops/nixos/blade-kim/default.nix delete mode 100644 ops/nixos/blade-paris/default.nix delete mode 100644 ops/nixos/blade-torres/default.nix delete mode 100644 ops/nixos/blade-tuvok/default.nix delete mode 100644 ops/nixos/lib/blade-router.nix delete mode 100644 ops/nixos/lib/blade.nix diff --git a/ops/nixos/blade-chakotay/default.nix b/ops/nixos/blade-chakotay/default.nix deleted file mode 100644 index 4acb86c5cb..0000000000 --- a/ops/nixos/blade-chakotay/default.nix +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01012b89ef8e36218414588461cda968a6fbfa5bf39e398aaeda43d9e398b05f86800000000000000000000097b1f26d001a240091558107b6a8e160-0:0"; - - # Networking! - networking = { - hostName = "blade-chakotay"; - hostId = "40bc5a75"; - }; - my.ip.tailscale = "100.121.11.7"; - my.blade.bay = 5; - my.blade.macAddress = { - internal = "e4:11:5b:ac:e3:b8"; - storage = "e4:11:5b:ac:e3:bc"; - }; - my.deploy.enable = false; - my.rundeck.expectedOnline = false; - - services.ceph = { - #osd.enable = true; - }; -} diff --git a/ops/nixos/blade-janeway/default.nix b/ops/nixos/blade-janeway/default.nix deleted file mode 100644 index 4b503ef925..0000000000 --- a/ops/nixos/blade-janeway/default.nix +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_0401c0556e165beb5c1df7b9f438e6ba98af3caa816d88e69b2bf7ec2909f6ca293a000000000000000000009c7343f0ff88671891558107c52824c1-0:0"; - - # Networking! - networking = { - hostName = "blade-janeway"; - hostId = "3a62390f"; - }; - my.ip.tailscale = "100.121.116.85"; - my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6279:7455"; - my.blade.bay = 3; - my.blade.macAddress = { - internal = "e8:39:35:1f:7f:8a"; - storage = "e8:39:35:1f:7f:8e"; - }; - - services.ceph = { - mon.enable = true; - mds.enable = true; - osd = { - enable = true; - daemons = [ "0" "1" ]; - }; - }; -} diff --git a/ops/nixos/blade-kim/default.nix b/ops/nixos/blade-kim/default.nix deleted file mode 100644 index 1abe214dfd..0000000000 --- a/ops/nixos/blade-kim/default.nix +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01015d3cbc98a05319a5ebb2b00fed57649bb0a8b5b5e517be322f1be132d4d8d60d00000000000000000000021d5c2c0083090091558107b6a8d5cd-0:0"; - - # Networking! - networking = { - hostName = "blade-kim"; - hostId = "1643efb6"; - }; - my.ip.tailscale = "100.84.36.62"; - my.blade.bay = 1; - my.blade.macAddress = { - internal = "e4:11:5b:ac:d1:7a"; - storage = "e4:11:5b:ac:d1:7e"; - }; - my.deploy.enable = false; - my.rundeck.expectedOnline = false; - - services.ceph = { - #osd.enable = true; - }; -} diff --git a/ops/nixos/blade-paris/default.nix b/ops/nixos/blade-paris/default.nix deleted file mode 100644 index c2958d5a23..0000000000 --- a/ops/nixos/blade-paris/default.nix +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade-router.nix - ../lib/blade.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-SanDisk_Ultra_4C530001320720111165-0:0"; - - # Networking! - networking = { - hostName = "blade-paris"; - hostId = "41b2a198"; - }; - my.ip.tailscale = "100.81.131.61"; - my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6251:833d"; - my.blade.onZFS = false; - my.blade.bay = 2; - my.blade.macAddress = { - internal = "e4:11:5b:ac:e4:8a"; - storage = "e4:11:5b:ac:e4:8e"; - internet = "e4:11:5b:ac:e4:8c"; - }; - my.deploy.enable = false; - - services.ceph = { - mon.enable = true; - osd = { - enable = true; - daemons = [ "2" ]; - }; - }; - - my.blade-router = { - addresses.linknet = { - v4 = { local = "195.74.55.23"; remote = "195.74.55.22"; }; - v6 = { - local = "2a03:ee40:8080:9:2::2"; - remote = "2a03:ee40:8080:9:2::1"; - }; - }; - - addresses.br-public = { - v4.addr = "92.118.28.253"; - v6.addr = "2a09:a441::fffe"; - }; - - vrrp.priority = 50; - }; -} diff --git a/ops/nixos/blade-torres/default.nix b/ops/nixos/blade-torres/default.nix deleted file mode 100644 index 7797b97e1d..0000000000 --- a/ops/nixos/blade-torres/default.nix +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_01011cbe2d1b827aedae2d5668fecf2a2dbdab7d0ecc0416f6e7c5485ecec8f5e5870000000000000000000080ce63590095210091558107b6a8e55d-0:0"; - - # Networking! - networking = { - hostName = "blade-torres"; - hostId = "86db1d9c"; - - # HACK: torres' secondary intf. seems to be having problems - # temporarily remap this onto the VLANs with everything else. - vlans.en-storage = { - id = 101; - interface = "en-int"; - }; - }; - my.ip.tailscale = "100.92.118.36"; - my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:625c:7624"; - my.blade.bay = 8; - my.blade.macAddress = { - internal = "e4:11:5b:ac:e3:cc"; - storage = null; - #storage = "e4:11:5b:ac:e3:d0"; - }; - - services.ceph = { - osd = { - enable = true; - daemons = [ "4" "5" ]; - }; - }; -} diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix deleted file mode 100644 index 77b6ef68d9..0000000000 --- a/ops/nixos/blade-tuvok/default.nix +++ /dev/null @@ -1,272 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/blade-router.nix - ../lib/blade.nix - ../lib/fup.nix - ]; - - boot.loader.grub.device = "/dev/disk/by-id/usb-SanDisk_Cruzer_Blade_00023903100122082002-0:0"; - - # Networking! - networking = { - hostName = "blade-tuvok"; - hostId = "525229f7"; - firewall.allowedTCPPorts = [ 80 443 ]; - firewall.allowedUDPPorts = [ - # Wireguard - 51820 51821 51822 - ]; - localCommands = '' - # Check if we already have our little minicontainer setup - ip netns list | grep -q wg-endpoint || ( - ip netns add wg-endpoint - ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint - ip link set dev ve-wg-endpoint master br-public - ip link set dev ve-wg-endpoint up - ip -n wg-endpoint link set dev ve-int up - ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int - ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int - ip -n wg-endpoint route add default via 92.118.28.1 - ip -n wg-endpoint route add default via 2a09:a441::1 - ip netns exec wg-endpoint ${pkgs.procps}/bin/sysctl net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.default.forwarding=1 - ) - - # We are forcing the MAC on vl-linx to be something particular, - # but we also need to add the corresponding LL address. - # This is cursed. - ip -6 addr add fe80::e611:5bff:feac:e400/64 dev vl-linx &>/dev/null || true - ''; - nat.extraCommands = '' - iptables -w -t nat -A nixos-nat-post -m mark --mark 0x1 -o vl-linx -j SNAT --to-source 92.118.28.1 - ''; - }; - my.ip.tailscale = "100.119.123.33"; - my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6277:7b21"; - my.blade.bay = 6; - my.blade.macAddress = { - internal = "e4:11:5b:ac:e3:fe"; - storage = "e4:11:5b:ac:e4:02"; - internet = "e4:11:5b:ac:e3:ff"; - public = "e4:11:5b:ac:e4:00"; - }; - - services.ceph = { - mon.enable = true; - osd = { - enable = true; - daemons = [ "3" ]; - }; - }; - - services.nginx = { - enable = true; - recommendedTlsSettings = true; - recommendedGzipSettings = true; - virtualHosts."objdump.zxcvbnm.ninja" = { - default = true; - forceSSL = true; - locations."/" = { - proxyPass = "http://localhost:7480"; - extraConfig = '' - proxy_redirect off; - client_max_body_size 0; - proxy_buffering off; - ''; - }; - }; - }; - my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = { - hostnames = [ "objdump.zxcvbnm.ninja" "*.objdump.zxcvbnm.ninja" ]; - nginxVirtualHosts = [ "objdump.zxcvbnm.ninja" ]; - }; - my.fup.listen = [ - "0.0.0.0" "[::]" - ]; - - my.blade-router = { - addresses.linknet = { - v4 = { local = "195.74.55.21"; remote = "195.74.55.20"; }; - v6 = { - local = "2a03:ee40:8080:9:1::2"; - remote = "2a03:ee40:8080:9:1::1"; - }; - }; - - addresses.br-public = { - v4.addr = "92.118.28.254"; - v6.addr = "2a09:a441::ffff"; - }; - - linx.enable = true; - - vrrp.priority = 100; - }; - - networking.wireguard = let - ifBase = { - listenPort = null; - allowedIPsAsRoutes = false; - socketNamespace = "wg-endpoint"; - }; - peerBase.allowedIPs = [ - "0.0.0.0/0" - "::/0" - ]; - swannPeerBase = peerBase // { - endpoint = null; # dynamic - publicKey = secrets.wireguard.tuvok-swann.swann.publicKey; - }; - swannBase = ifBase // { - privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey; - peers = [swannPeerBase]; - }; - in { - enable = true; - interfaces.wg-swann-ee = swannBase // { - ips = [ - "2a09:a442::2:2/112" - "92.118.30.3/31" - ]; - listenPort = 51821; - }; - interfaces.wg-swann-gnet = swannBase // { - ips = [ - "2a09:a442::3:2/112" - "92.118.30.5/31" - ]; - listenPort = 51822; - peers = [(swannPeerBase // { - endpoint = "185.250.189.204:51822"; - })]; - }; - }; - environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile { - name = "bird-wg-endpoint.conf"; - text = '' - router id 92.118.28.252; - protocol kernel { - persist; - ipv4 { - import none; - export all; - }; - }; - protocol kernel { - persist; - ipv6 { - import none; - export all; - }; - }; - protocol device {}; - - protocol static export4 { - ipv4 {}; - route 92.118.30.0/24 via 92.118.30.2 bfd { - # EE - preference = 10; - }; - route 92.118.30.0/24 via 92.118.30.4 bfd { - # GNetwork - preference = 200; - }; - }; - protocol static export6 { - ipv6 {}; - route 2a09:a443::/32 via 2a09:a442::1:1 bfd { - # Virgin Media - preference = 100; - }; - route 2a09:a443::/32 via 2a09:a442::2:1 bfd { - # EE - preference = 10; - }; - route 2a09:a443::/32 via 2a09:a442::3:1 bfd { - # GNetwork - preference = 200; - }; - }; - - protocol bfd { - interface "*" { - min rx interval 10ms; - min tx interval 50ms; - idle tx interval 1s; - multiplier 20; - }; - neighbor 92.118.30.2; - neighbor 2a09:a442::2:1; - neighbor 92.118.30.4; - neighbor 2a09:a442::3:1; - }; - ''; - checkPhase = '' - ${pkgs.bird2}/bin/bird -d -p -c $out - ''; - }; - systemd.services.bird-wg-endpoint = { - wantedBy = [ "multi-user.target" ]; - reloadIfChanged = true; - - description = "BIRD inside wg-endpoint netns"; - after = [ "network.target" ]; - restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ]; - - serviceConfig = { - Type = "forking"; - Restart = "on-failure"; - CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID" - # see bird/sysdep/linux/syspriv.h - "CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ]; - ProtectSystem = "full"; - ProtectHome = "yes"; - SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io"; - MemoryDenyWriteExecute = "yes"; - - ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down"; - ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl"; - ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'"; - NetworkNamespacePath = "/var/run/netns/wg-endpoint"; - }; - }; - # It is at this point he realises it would've been easier if he'd used NixOS containers instead. - systemd.services.bird-wg-endpoint-exporter = { - after = [ "bird-wg-endpoint.service" "network.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - DynamicUser = true; - User = "bird-exporter"; - Group = "bird-exporter"; - PrivateTmp = true; - Restart = "always"; - WorkingDirectory = "/tmp"; - SupplementaryGroups = "bird2"; - ExecStart = '' - ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \ - -web.listen-address 0.0.0.0:9325 \ - -bird.socket /var/run/bird-wg-endpoint.ctl \ - -bird.v2=true \ - -format.new=true - ''; - }; - }; - my.prometheus.additionalExporterPorts.bird-wg-endpoint-exporter = 9325; - services.lukegbgp.config.export = { - v4Extra = '' - route 92.118.30.0/24 via 92.118.28.252; - ''; - v6Extra = '' - route 2a09:a443::/32 via 2a09:a441::f00f; - ''; - }; - systemd.services.wireguard-wg-swann-ee.after = lib.mkAfter [ "network-local-commands.service" ]; - systemd.services.wireguard-wg-swann-gnet.after = lib.mkAfter [ "network-local-commands.service" ]; - -} diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix index 3c882dfb67..97d312d228 100644 --- a/ops/nixos/default.nix +++ b/ops/nixos/default.nix @@ -26,12 +26,6 @@ let "swann" "clouvider-lon01" "etheroute-lon01" - "blade-janeway" - "blade-tuvok" - "blade-paris" - "blade-torres" - "blade-chakotay" - "blade-kim" "frantech-lux01" "frantech-nyc01" "frantech-las01" diff --git a/ops/nixos/lib/bgp.nix b/ops/nixos/lib/bgp.nix index d20d29772c..880a46e575 100644 --- a/ops/nixos/lib/bgp.nix +++ b/ops/nixos/lib/bgp.nix @@ -359,8 +359,8 @@ in { bgp_ext_community.add((ro, 205479, 5006)); bgp_ext_community.add((ro, 205479, 5007)); bgp_ext_community.add((ro, 205479, 5008)); - bgp_ext_community.add((ro, 205479, 5009)); # fastly from blade-tuvok - bgp_ext_community.add((ro, 205479, 5010)); # ovh from blade-tuvok + bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar + bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server # do not export to clouvider; they do... strange things. @@ -382,19 +382,19 @@ in { bgp_ext_community.add((ro, 205479, 2003)); # bgp_ext_community.add((ro, 205479, 3000)); # clouvider from clouvider-lon01 bgp_ext_community.add((ro, 205479, 4000)); # frantech from frantech-nyc01/veloxserv from etheroute-lon01 - bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from blade-tuvok/blade-paris + bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from rexxar bgp_ext_community.add((ro, 205479, 4002)); # mercury from etheroute-lon01 - bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from blade-tuvok - bgp_ext_community.add((ro, 205479, 5001)); # linx route server from blade-tuvok - bgp_ext_community.add((ro, 205479, 5002)); # facebook from blade-tuvok - bgp_ext_community.add((ro, 205479, 5003)); # openpeering from blade-tuvok - bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from blade-tuvok - bgp_ext_community.add((ro, 205479, 5005)); # he from blade-tuvok - bgp_ext_community.add((ro, 205479, 5006)); # clouvider from blade-tuvok - bgp_ext_community.add((ro, 205479, 5007)); # google from blade-tuvok - bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from blade-tuvok - bgp_ext_community.add((ro, 205479, 5009)); # fastly from blade-tuvok - bgp_ext_community.add((ro, 205479, 5010)); # ovh from blade-tuvok + bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from rexxar + bgp_ext_community.add((ro, 205479, 5001)); # linx route server from rexxar + bgp_ext_community.add((ro, 205479, 5002)); # facebook from rexxar + bgp_ext_community.add((ro, 205479, 5003)); # openpeering from rexxar + bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from rexxar + bgp_ext_community.add((ro, 205479, 5005)); # he from rexxar + bgp_ext_community.add((ro, 205479, 5006)); # clouvider from rexxar + bgp_ext_community.add((ro, 205479, 5007)); # google from rexxar + bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from rexxar + bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar + bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server # do not export to clouvider; they do... strange things. diff --git a/ops/nixos/lib/blade-router.nix b/ops/nixos/lib/blade-router.nix deleted file mode 100644 index 6534109cc2..0000000000 --- a/ops/nixos/lib/blade-router.nix +++ /dev/null @@ -1,417 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ lib, config, pkgs, ... }: -with lib; -let - cfg = config.my.blade-router; -in -{ - imports = [ - ../lib/bgp.nix - ]; - - options.my.blade-router = { - addresses.linknet.v4 = { - local = mkOption { type = types.str; }; - remote = mkOption { type = types.str; }; - prefixLength = mkOption { type = types.int; default = 31; }; - }; - addresses.linknet.v6 = { - local = mkOption { type = types.str; }; - remote = mkOption { type = types.str; }; - prefixLength = mkOption { type = types.int; default = 126; }; - }; - - addresses.br-public.v4 = { - addr = mkOption { type = types.str; }; - prefixLength = mkOption { type = types.int; default = 24; }; - }; - addresses.br-public.v6 = { - addr = mkOption { type = types.str; }; - prefixLength = mkOption { type = types.int; default = 48; }; - }; - - addresses.br-public-vip.v4 = { - addr = mkOption { type = types.str; default = "92.118.28.1"; }; - prefixLength = mkOption { type = types.int; default = 24; }; - }; - addresses.br-public-vip.v6 = { - addr = mkOption { type = types.str; default = "2a09:a441::1"; }; - prefixLength = mkOption { type = types.int; default = 48; }; - }; - addresses.br-public-vip.v6-ll = { - addr = mkOption { type = types.str; default = "fe80::f00f"; }; - prefixLength = mkOption { type = types.int; default = 64; }; - }; - - addresses.br-public-radvd-prefix = { - addr = mkOption { type = types.str; default = "2a09:a441:0:ffff::"; }; - prefixLength = mkOption { type = types.int; default = 64; }; - }; - - linx.enable = mkEnableOption "linx"; - addresses.vl-linx.v4 = { - addr = mkOption { type = types.str; default = "195.66.224.58"; }; - prefixLength = mkOption { type = types.int; default = 21; }; - }; - addresses.vl-linx.v6 = { - addr = mkOption { type = types.str; default = "2001:7f8:4::3:22a7:1"; }; - prefixLength = mkOption { type = types.int; default = 64; }; - }; - - vrrp.priority = mkOption { type = types.int; }; - }; - - config = { - boot.kernel.sysctl = { - "net.ipv4.ip_forward" = 1; - "net.ipv6.conf.all.forwarding" = 1; - - "net.ipv4.conf.vl-linx.arp_announce" = 1; - "net.ipv4.conf.vl-linx.arp_ignore" = 1; - "net.ipv4.neigh.vl-linx.base_reachable_time_ms" = 14400000; - "net.ipv6.neigh.vl-linx.base_reachable_time_ms" = 14400000; - }; - - networking = { - vlans.vl-transit = { - id = 100; - interface = "en-internet"; - }; - vlans.vl-linx = { - id = 200; - interface = "en-internet"; - }; - - interfaces.br-public.ipv4.addresses = [{ - address = config.my.blade-router.addresses.br-public.v4.addr; - prefixLength = config.my.blade-router.addresses.br-public.v4.prefixLength; - }]; - interfaces.br-public.ipv6.addresses = [{ - address = config.my.blade-router.addresses.br-public.v6.addr; - prefixLength = config.my.blade-router.addresses.br-public.v6.prefixLength; - }]; - interfaces.vl-transit.ipv4.addresses = [{ - address = config.my.blade-router.addresses.linknet.v4.local; - prefixLength = config.my.blade-router.addresses.linknet.v4.prefixLength; - }]; - interfaces.vl-transit.ipv6.addresses = [{ - address = config.my.blade-router.addresses.linknet.v6.local; - prefixLength = config.my.blade-router.addresses.linknet.v6.prefixLength; - }]; - interfaces.vl-linx.macAddress = "e4:11:5b:ac:e4:00"; - interfaces.vl-linx.ipv4.addresses = [{ - address = cfg.addresses.vl-linx.v4.addr; - prefixLength = cfg.addresses.vl-linx.v4.prefixLength; - }]; - interfaces.vl-linx.ipv6.addresses = [{ - address = cfg.addresses.vl-linx.v6.addr; - prefixLength = cfg.addresses.vl-linx.v6.prefixLength; - }]; - defaultGateway = config.my.blade-router.addresses.linknet.v4.remote; - defaultGateway6 = config.my.blade-router.addresses.linknet.v6.remote; - firewall.extraCommands = '' - iptables -A INPUT -p vrrp -i br-mgmt -j ACCEPT - ip6tables -A INPUT -p vrrp -i br-mgmt -j ACCEPT - ''; - }; - - services.lukegbgp = { - enable = true; - config = let - linx = { - local = { - asn = 205479; - v4 = cfg.addresses.vl-linx.v4.addr; - v6 = cfg.addresses.vl-linx.v6.addr; - }; - }; - in { - local.routerID = config.my.blade-router.addresses.linknet.v4.local; - peering.veloxserv = { - local = { - asn = 205479; - v4 = config.my.blade-router.addresses.linknet.v4.local; - v6 = config.my.blade-router.addresses.linknet.v6.local; - }; - remote = { - asn = 3170; - export_community = 4001; - bgp_local_pref = 101; - drop_asns = [ - 15169 # prefer RS to transit - ]; - routers = [{ - v4 = config.my.blade-router.addresses.linknet.v4.remote; - v6 = config.my.blade-router.addresses.linknet.v6.remote; - }]; - }; - }; - peering.bgptoolscollector = { - local = { - asn = 205479; - v4 = config.my.blade-router.addresses.linknet.v4.local; - v6 = config.my.blade-router.addresses.linknet.v6.local; - }; - remote = { - asn = 212232; - export_community = 5000; - routers = [{ - enabled = cfg.linx.enable; - v4 = "185.230.223.42"; - v6 = "2a0c:2f07:9459::b1"; - multihop = 64; - }]; - prefix_limit.v4 = 0; - prefix_limit.v6 = 0; - is_route_collector = true; - }; - }; - peering.linxcollector = linx // { - remote = { - asn = 5459; - export_community = 5000; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.224.254"; - v6 = "2001:7f8:4::1553:1"; - }]; - prefix_limit.v4 = 0; - prefix_limit.v6 = 0; - is_route_collector = false; # don't make people sad. - }; - }; - peering.linx = linx // { - remote = { - asn = 8714; - export_community = 5001; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.230"; - v6 = "2001:7f8:4::220a:1"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.225.231"; - v6 = "2001:7f8:4::220a:2"; - }]; - bgp_local_pref = 109; - must_be_next_hop = false; - }; - }; - peering.facebook = linx // { - remote = { - asn = 32934; - export_community = 5002; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.69"; - v6 = "2001:7f8:4::80a6:1"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.225.121"; - v6 = "2001:7f8:4::80a6:2"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.227.19"; - v6 = "2001:7f8:4::80a6:5"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.226.140"; - v6 = "2001:7f8:4::80a6:3"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 100; - prefix_limit.v6 = 100; - }; - }; - peering.openpeering = linx // { - remote = { - asn = 20562; - export_community = 5003; - passive = true; # pending - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.53"; - v6 = "2001:7f8:4::5052:1"; - }]; - bgp_local_pref = 110; - prefix_limit.v4 = 16000; - prefix_limit.v6 = 3000; - }; - }; - peering.freetransitnet = linx // { - remote = { - asn = 212895; - export_community = 5004; - passive = true; # pending v6 - bgp_local_pref = 100; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.105"; - v6 = "2001:7f8:4::3:3f9f:2"; - }]; - }; - }; - peering.he = linx // { - remote = { - asn = 6939; - export_community = 5005; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.224.21"; - v6 = "2001:7f8:4::1b1b:1"; - }]; - bgp_local_pref = 108; - prefix_limit.v4 = 176000; - prefix_limit.v6 = 156000; - }; - }; - peering.clouvider = linx // { - remote = { - asn = 62240; - export_community = 5006; - passive = true; # pending - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.227.14"; - v6 = "2001:7f8:4::f320:1"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 1000; - prefix_limit.v6 = 1000; - }; - }; - peering.google = linx // { - remote = { - asn = 15169; - export_community = 5007; - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.224.125"; - v6 = "2001:7f8:4::3b41:1"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 15000; - prefix_limit.v6 = 10000; - }; - }; - peering.cloudflare = linx // { - remote = { - asn = 13337; - export_community = 5008; - passive = true; # pending - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.179"; - v6 = "2001:7f8:4::3417:1"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.227.207"; - v6 = "2001:7f8:4::3417:2"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 20000; - prefix_limit.v6 = 2000; - }; - }; - peering.fastly = linx // { - remote = { - asn = 54113; - export_community = 5009; - passive = true; # pending - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.225.91"; - v6 = "2001:7f8:4::d361:1"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.227.114"; - v6 = "2001:7f8:4::d361:2"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 250; - prefix_limit.v6 = 250; - }; - }; - peering.ovh = linx // { - remote = { - asn = 16276; - export_community = 5010; - passive = true; # pending - routers = [{ - enabled = cfg.linx.enable; - v4 = "195.66.224.220"; - v6 = "2001:7f8:4::3f94:1"; - } { - enabled = cfg.linx.enable; - v4 = "195.66.225.6"; - v6 = "2001:7f8:4::3f94:2"; - }]; - bgp_local_pref = 120; - prefix_limit.v4 = 1000; - prefix_limit.v6 = 200; - }; - }; - export.v4 = [ "92.118.28.0/24" ]; - export.v6 = [ "2a09:a441::/32" ]; - }; - }; - - services.keepalived = let - mgmtBase = { - interface = "br-mgmt"; - state = "MASTER"; - priority = config.my.blade-router.vrrp.priority; - }; - in { - enable = true; - vrrpInstances.mgmtGateway4 = mgmtBase // { - virtualIps = [ - { addr = "10.100.0.1/23"; } - { addr = "${config.my.blade-router.addresses.br-public-vip.v4.addr}/${toString config.my.blade-router.addresses.br-public-vip.v4.prefixLength}"; dev = "br-public"; } - ]; - virtualRouterId = 1; - }; - vrrpInstances.mgmtGateway6 = mgmtBase // { - virtualIps = [ - { addr = "${config.my.blade-router.addresses.br-public-vip.v6-ll.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6-ll.prefixLength}"; dev = "br-public"; } - { addr = "${config.my.blade-router.addresses.br-public-vip.v6.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6.prefixLength}"; dev = "br-public"; } - ]; - virtualRouterId = 2; - }; - extraGlobalDefs = '' - enable_script_security - script_user root - ''; - extraConfig = '' - vrrp_sync_group mgmtGateway { - group { - mgmtGateway4 - mgmtGateway6 - } - } - ''; - }; - - services.radvd = { - enable = true; - config = '' - interface br-public { - AdvSendAdvert on; - MinRtrAdvInterval 30; - MaxRtrAdvInterval 100; - AdvRASrcAddress { - ${config.my.blade-router.addresses.br-public-vip.v6-ll.addr}; - }; - prefix ${config.my.blade-router.addresses.br-public-radvd-prefix.addr}/${toString config.my.blade-router.addresses.br-public-radvd-prefix.prefixLength} { - AdvOnLink on; - AdvAutonomous on; - AdvRouterAddr off; - }; - }; - ''; - }; - }; -} diff --git a/ops/nixos/lib/blade.nix b/ops/nixos/lib/blade.nix deleted file mode 100644 index da7e49d68e..0000000000 --- a/ops/nixos/lib/blade.nix +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Luke Granger-Brown -# -# SPDX-License-Identifier: Apache-2.0 - -{ depot, lib, pkgs, rebuilder, config, ... }: -let - inherit (depot.ops) secrets; -in { - imports = [ - ../lib/minimal.nix - ../lib/zfs.nix - ]; - - options.my.blade = { - bay = lib.mkOption { - type = lib.types.int; - }; - macAddress.internal = lib.mkOption { - type = lib.types.str; - }; - macAddress.storage = lib.mkOption { - type = lib.types.nullOr lib.types.str; - }; - macAddress.internet = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - }; - macAddress.public = lib.mkOption { - # If not using a VLAN. - type = lib.types.nullOr lib.types.str; - default = null; - }; - - onZFS = lib.mkOption { type = lib.types.bool; default = true; }; - }; - - config = { - boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; - boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ]; - - # Enable serial console. - boot.loader.grub.extraConfig = '' - serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1 - terminal_input console serial - terminal_output console serial - ''; - boot.kernelParams = [ - "console=tty1" - "console=ttyS0,115200" # <-- /dev/console - - "mitigations=off" - ]; - - my.rundeck.tags = [ "blade" ]; - - fileSystems = let - zfs = device: { - device = device; - fsType = "zfs"; - }; - tmpfs = size: { - device = "none"; - fsType = "tmpfs"; - options = [ "defaults" "size=${size}" "mode=755" ]; - }; - in { - "/boot" = { - device = "/dev/disk/by-label/boot"; - fsType = "ext4"; - }; - "/var/log" = tmpfs "2G"; - "/var/cache" = tmpfs "16G"; - } // (lib.optionalAttrs (config.my.blade.onZFS) { - "/" = zfs "tank/local/root"; - "/tmp" = zfs "tank/local/tmp"; - "/nix" = zfs "tank/local/nix"; - "/var" = zfs "tank/safe/var"; - "/home" = zfs "tank/safe/home"; - }) // (lib.optionalAttrs (!config.my.blade.onZFS) { - "/" = { - device = "/dev/disk/by-label/root"; - fsType = "ext4"; - }; - }) // (lib.optionalAttrs (config.services.ceph.osd.enable || config.services.ceph.mgr.enable || config.services.ceph.mon.enable || config.services.ceph.mgr.enable) { - "/var/lib/ceph" = { - device = "/dev/disk/by-label/var-lib-ceph"; - fsType = "xfs"; - }; - }); - - boot.loader.grub.enable = true; - - # Networking! - networking = { - domain = "blade.as205479.net"; - nameservers = ["8.8.8.8" "8.8.4.4"]; - search = lib.mkBefore [ - "blade.as205479.net" - "storage.blade.as205479.net" - ]; - bridges = let - br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; }; - in { - br-mgmt = br [ "en-int" ]; - br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ]; - }; - vlans = ({} // - (if config.my.blade.macAddress.public == null then { - vl-int-public = { - id = 100; - interface = "en-int"; - }; - } else {})); - - interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{ - address = "10.100.0.${toString (100 + config.my.blade.bay)}"; - prefixLength = 23; - }]; - interfaces.en-storage.ipv4.addresses = lib.mkBefore [{ - address = "10.100.2.${toString (100 + config.my.blade.bay)}"; - prefixLength = 24; - }]; - - defaultGateway = lib.mkDefault "10.100.0.1"; - - firewall.allowedUDPPorts = [ - 41641 # Tailscale - ]; - firewall.interfaces.en-storage.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ]; - firewall.interfaces.en-storage.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }]; - - firewall.extraCommands = '' - iptables -A nixos-fw -i en-storage -s 10.100.2.0/23 -j ACCEPT - iptables -A nixos-fw -i br-mgmt -s 10.100.0.0/23 -j ACCEPT - ''; - - nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) { - enable = true; - internalInterfaces = [ "br-mgmt" ]; - externalInterface = "vl-transit"; - externalIP = "92.118.28.1"; - }; - }; - services.udev.extraRules = '' - ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int" - '' + (lib.optionalString (config.my.blade.macAddress.storage != null) '' - ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage" - '') + (lib.optionalString (config.my.blade.macAddress.internet != null) '' - ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet" - '') + (lib.optionalString (config.my.blade.macAddress.public != null) '' - ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public" - ''); - - environment.systemPackages = with pkgs; [ - ceph - xfsprogs - ]; - - services.ceph = { - enable = true; - global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; - global.monHost = "10.100.2.103, 10.100.2.106, 10.100.2.102"; - global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris"; - global.publicNetwork = "10.100.2.0/24"; - global.clusterNetwork = "10.100.2.0/24"; - extraConfig.rgw_dns_name = "objdump.zxcvbnm.ninja"; - extraConfig.rgw_data_log_backing = "omap"; - extraConfig.rgw_default_data_log_backing = "omap"; - mon.daemons = [ config.networking.hostName ]; - mds.daemons = [ config.networking.hostName ]; - rgw.daemons = [ config.networking.hostName ]; - mgr.daemons = [ config.networking.hostName ]; - mgr.enable = config.services.ceph.mon.enable; - rgw.enable = true; - client.enable = true; - client.extraConfig = { - "client.libvirt" = { - rbd_cache = "true"; - rbd_cache_policy = "writeback"; - rbd_cache_size = "2Gi"; - rbd_cache_max_dirty = "1792Mi"; - rbd_cache_target_dirty = "128Mi"; - }; - }; - }; - systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable { - enable = true; - description = "Ceph OSD pre-start"; - before = [ "network-online.target" "ceph-osd.target" ]; - wantedBy = [ "ceph-osd.target" ]; - - path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; - - serviceConfig = { - Type = "oneshot"; - ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; - }; - }; - - virtualisation.libvirtd = { - enable = true; - qemu = { - runAsRoot = true; - package = pkgs.qemu.override { - gtkSupport = false; - sdlSupport = false; - spiceSupport = true; - cephSupport = true; - smartcardSupport = false; - pulseSupport = false; - alsaSupport = false; - libiscsiSupport = false; - - hostCpuOnly = true; - }; - }; - package = pkgs.libvirt.override { - enableCeph = true; - enableIscsi = true; - }; - }; - security.polkit.enable = true; - users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ]; - - # Our disk is slow; don't write to it... - services.journald.extraConfig = '' - Storage=volatile - ''; - systemd.coredump.extraConfig = '' - Storage=none - ProcessSizeMax=0 - ''; - - system.stateVersion = "21.05"; - }; -} diff --git a/ops/nixos/totoro/default.nix b/ops/nixos/totoro/default.nix index 8621649caa..e8c10f1267 100644 --- a/ops/nixos/totoro/default.nix +++ b/ops/nixos/totoro/default.nix @@ -287,25 +287,6 @@ in { }]; globalConfig.scrape_interval = "15s"; scrapeConfigs = (builtins.attrValues depot.ops.nixos.systemExporters) ++ [{ - job_name = "blade-oa/snmp"; - metrics_path = "/snmp"; - params = { - module = ["hpe"]; - }; - static_configs = [{ - targets = ["10.100.1.200"]; - }]; - relabel_configs = [{ - source_labels = ["__address__"]; - target_label = "__param_target"; - } { - source_labels = ["__param_target"]; - target_label = "instance"; - } { - target_label = "__address__"; - replacement = "totoro:${toString config.services.prometheus.exporters.snmp.port}"; - }]; - } { job_name = "minotar/minotarproxy"; scheme = "https"; static_configs = [{ @@ -343,33 +324,9 @@ in { rules: # Blade power - - alert: AveragePowerUsageTooHigh - expr: (sum(avg_over_time(cpqRackPowerSupplyCurPwrOutput{job="blade-oa/snmp"}[10m])) / 230) > 6.5 - labels: - severity: page - annotations: - summary: "Blade: Power Usage Too High (rolling)" - description: "Power usage of blade system has been too high for last 10 minutes ({{ $value }}). https://grafana.int.lukegb.com/d/g-u3XQ8Gk/blade-power" - - alert: PowerUsageTooHigh - expr: (sum(cpqRackPowerSupplyCurPwrOutput{job="blade-oa/snmp"}) / 230) > 6.5 - for: 10m - labels: - severity: page - annotations: - summary: "Blade: Power Usage Too High" - description: "Power usage of blade system has been too high for last 10 minutes ({{ $value }}). https://grafana.int.lukegb.com/d/g-u3XQ8Gk/blade-power" - - alert: BladePowerUsageOutOfBounds - expr: node_hwmon_power_average_watt{system=~"blade-.*"} > on () group_left() (1.5 * quantile(0.5, node_hwmon_power_average_watt{system=~"blade-.*"})) - for: 60m - labels: - severity: page - annotations: - summary: "Blade: Single Blade Power Usage Out of Bounds" - description: "{{ $labels.system }} has power usage of {{ $value }}, which is out of expected bounds." - # Systems - alert: NodeExporterDown - expr: up{exporter="node", system=~"(blade-(tuvok|paris|janeway|torres)|kusakabe|swann|totoro|clouvider-.*|etheroute-.*|bvm-.*)"} < 1 + expr: up{exporter="node", system=~"(rexxar|kusakabe|swann|totoro|clouvider-.*|etheroute-.*|bvm-.*)"} < 1 for: 30m labels: severity: page