depot/ops/nixos/swann/default.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, config, ... }:
let
  inherit (depot.ops) secrets;
in {
  imports = [
    # We include this just so it sets some sysctls and firewall settings.
    ../lib/bgp.nix
  ];

  boot.initrd.availableKernelModules = [
    "sd_mod"
    "ahci"
    "usb_storage"
    "usbhid"
  ];
  boot.kernelParams = [ "mitigations=off" ];

  fileSystems = {
    "/" = {
      device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
      fsType = "ext4";
    };
    "/boot" = {
      device = "/dev/disk/by-uuid/AB36-5BE4";
      fsType = "vfat";
    };
  };

  nix.settings.max-jobs = lib.mkDefault 4;

  # Use the systemd-boot EFI boot loader.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  # Networking!
  networking = {
    # Routing tables:
    # bgp (150) -- contains default routes over WG tunnels
    # wg-vm (151) -- contains default routes over WG tunnels
    # wg-ee (152) -- contains default routes over WG tunnels
    # wg-gnet (153) -- contains default routes over WG tunnels
    # ee (201) -- table contains a default route via EE
    # vm (202) -- table contains a default route via VM
    # gnet (203) -- table contains a default route via gnetwork
    # main (254) -- basically empty

    hostName = "swann"; # Define your hostname.
    domain = "int.as205479.net";
    nameservers = ["8.8.8.8" "8.8.4.4"];
    useNetworkd = true;
    interfaces = {
      lo = {
        ipv4.addresses = [
          { address = "127.0.0.1"; prefixLength = 8; }
          { address = "92.118.30.254"; prefixLength = 32; }
          { address = "92.118.30.253"; prefixLength = 32; }
        ];
      };
      en-virginmedia = {
        useDHCP = true;
        macAddress = "e4:3a:6e:16:07:61";
      };
      en-gnet = {
        useDHCP = true;
        # Additional options configured in networkd.
      };
      en-ee = {
        useDHCP = true;
        # Additional options configured in networkd.
      };
      en-general = {
        ipv4.addresses = [
          { address = "192.168.1.1"; prefixLength = 23; }
          { address = "92.118.30.17"; prefixLength = 28; }
        ];
        ipv6.addresses = [
          { address = "2a09:a443::1"; prefixLength = 64; }
          { address = "2a09:a443:1::1"; prefixLength = 48; }
        ];
      };
      vl-eduroam = {
        ipv4.addresses = [
          { address = "192.168.10.1"; prefixLength = 24; }
        ];
        ipv6.addresses = [
          { address = "2a09:a443:2::1"; prefixLength = 64; }
          { address = "2a09:a443:3::1"; prefixLength = 48; }
        ];
      };
    };
    vlans = {
      vl-eduroam = {
        id = 100;
        interface = "en-general";
      };
    };

    localCommands = let
      claimedPriorities = { min = 10000; max = 10100; };
      rules = [
        # Route traffic to EE via WG... via EE.
        { priority = 10000; both = "fwmark 0xdead table 201"; }
        # Route traffic to VM via WG... via VM DHCP in table 202.
        { priority = 10001; both = "fwmark 0xbeef table 202"; }
        # Route traffic to GNetwork via WG... via DHCP in table 203.
        { priority = 10002; both = "fwmark 0xcafe table 203"; }

        # Make ping work over the tunnels.
        { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
        { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
        { priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; }

        # Now some subset of RFC1918 via main table too.
        { priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
        { priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
        { priority = 10022; v4 = "to 172.16.0.0/12 table main"; }

        # And the linknets...
        { priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
        { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
        { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
        { priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; }

        # add-on.ee.co.uk goes via EE.
        { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }

        # Anything originating from 192.168.200.0/24 should go via EE too.
        { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }

        # Everything else over WG.
        { priority = 10080; both = "table 150"; }

        # Fallbacks via GNetwork, VM, EE
        # Sometimes this seems to be required to get things moving, for some super unclear reason.
        { priority = 10090; both = "table 203"; }
        { priority = 10091; both = "table 202"; }
        { priority = 10092; both = "table 201"; }
      ];
      clearRules = map (x: ''
        ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
        ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
      '') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
      ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
        assert (both == "" || (v4 == "" && v6 == ""));
        assert priority >= claimedPriorities.min;
        assert priority < claimedPriorities.max;
        let
          rv4 = if v4 != "" then v4 else both;
          rv6 = if v6 != "" then v6 else both;
        in ''
          ${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
          ${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
        '';
      addRules = map ruleToLine rules;
    in ''
      # Fix Tailscale, by adding routing rules just before the one they add at prio 5200.
      ip -4 rule del priority 5196 || true
      ip -4 rule del priority 5197 || true
      ip -4 rule del priority 5198 || true
      ip -4 rule del priority 5199 || true
      ip -4 rule add from all fwmark 0x80000 lookup 150 priority 5196
      ip -4 rule add from all fwmark 0x80000 lookup 151 priority 5197
      ip -4 rule add from all fwmark 0x80000 lookup 152 priority 5198
      ip -4 rule add from all fwmark 0x80000 lookup 153 priority 5199

      ${lib.concatStringsSep "\n" clearRules}
      ${lib.concatStringsSep "\n" addRules}

      ip -4 route flush table 151 >/dev/null 2>&1 || true
      ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
      ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
      ip -6 route flush table 151 >/dev/null 2>&1 || true
      ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151
      ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151

      ip -4 route flush table 152 >/dev/null 2>&1 || true
      ip -4 route add 92.118.30.2/31 dev wg-tuvok-ee table 152
      ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
      ip -6 route flush table 152 >/dev/null 2>&1 || true
      ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
      ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152

      ip -4 route flush table 153 >/dev/null 2>&1 || true
      ip -4 route add 92.118.30.4/31 dev wg-tuvok-gnet table 153
      ip -4 route add default via 92.118.30.5 dev wg-tuvok-gnet table 153
      ip -6 route flush table 153 >/dev/null 2>&1 || true
      ip -6 route add 2a09:a442::3:0/112 dev wg-tuvok-gnet table 153
      ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153
    '';
  };
  systemd.network = {
    enable = true;
    networks."40-en-ee".dhcpV4Config.RouteTable = 201;
    networks."40-en-ee".linkConfig.RequiredForOnline = "no";
    networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202;
    networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no";
    networks."40-en-gnet".dhcpV4Config.RouteTable = 203;
  };
  my.ip.tailscale = "100.102.224.95";
  services.udev.extraRules = ''
    ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
    ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
    ATTR{address}=="e4:3a:6e:16:07:64", NAME="en-gnet"
    ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
  '';
  boot.kernel.sysctl = {
    "net.ipv4.ip_forward" = "1";
    "net.ipv6.conf.default.forwarding" = "1";
    "net.ipv6.conf.all.forwarding" = "1";
    "net.ipv6.conf.en-virginmedia.accept_ra" = "2";
    "net.ipv6.conf.en-ee.accept_ra" = "2";
    "net.ipv6.conf.en-gnet.accept_ra" = "2";
  };
  networking.nat = {
    enable = true;
    internalInterfaces = ["en-general"];
    externalInterface = "en-virginmedia";
    extraCommands = ''
      # Send PS5 RTMP to totoro instead.
      # See DHCP static lease.
      iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40

      # NAT packets going over EE plain.
      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE

      # NAT packets going over GNetwork plain.
      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE

      # SNAT packets we're sending over tunnels.
      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254

      # eduroam
      # > mark incoming eduroam packets
      iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2
      # > NAT packets going out directly.
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-virginmedia -j MASQUERADE
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE
      # > NAT packets going over tunnels.
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.253
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253
      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253
    '';
  };
  services.dhcpd4 = {
    enable = true;
    interfaces = ["en-general" "vl-eduroam"];
    authoritative = true;
    extraConfig = ''
      shared-network int {
        default-lease-time 600;
        max-lease-time 3600;
        option interface-mtu 1420;  # Wireguard

        subnet 192.168.1.0 netmask 255.255.255.0 {
          option subnet-mask 255.255.255.0;
          option routers 192.168.1.1;
          option domain-name-servers 192.168.1.1;
          option domain-name "house.as205479.net";

          range 192.168.1.100 192.168.1.200;
        }

        subnet 92.118.30.16 netmask 255.255.255.240 {
          option subnet-mask 255.255.255.240;
          option routers 92.118.30.17;
          option domain-name-servers 92.118.30.17;
          option domain-name "house-ext.as205479.net";
        }
      }

      subnet 192.168.10.0 netmask 255.255.255.0 {
        option subnet-mask 255.255.255.0;
        option routers 192.168.10.1;
        option domain-name-servers 192.168.10.1;
        option domain-name "eduroam.as205479.net";
        default-lease-time 600;
        max-lease-time 3600;
        option interface-mtu 1420;  # Wireguard

        range 192.168.10.100 192.168.10.200;
      }
    '';
    machines = [
      {
        hostName = "totoro";
        ethernetAddress = "40:8d:5c:1f:e8:68";
        ipAddress = "192.168.1.40";
      }
      {
        hostName = "totoro-pfsense";
        ethernetAddress = "52:54:00:cf:cd:94";
        ipAddress = "192.168.1.41";
      }
      {
        hostName = "kvm";
        ethernetAddress = "00:0d:5d:1b:14:ba";
        ipAddress = "192.168.1.50";
      }
      {
        hostName = "printer-xerox";
        ethernetAddress = "9c:93:4e:ad:1f:7b";
        ipAddress = "192.168.1.51";
      }
      {
        hostName = "ps5";
        ethernetAddress = "bc:33:29:26:01:5c";
        # This is used for DNAT on RTMP, above.
        ipAddress = "92.118.30.18";
      }
    ];
  };
  networking.wireguard = let
    ifBase = {
      listenPort = null;
      allowedIPsAsRoutes = false;
    };
    peerBase = {
      allowedIPs = [
        "0.0.0.0/0"
        "::/0"
      ];
    };
  in {
    enable = true;
    interfaces.wg-tuvok-vm = ifBase // {
      ips = [
        "2a09:a442::1:1/112"
        "92.118.30.0/31"
      ];
      listenPort = 51820;
      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
      peers = [(peerBase // {
        endpoint = "92.118.28.252:51820";
        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
      })];
      postSetup = ''
        wg set wg-tuvok-vm fwmark 0xbeef
      '';
    };
    interfaces.wg-tuvok-ee = ifBase // {
      ips = [
        "2a09:a442::2:1/112"
        "92.118.30.2/31"
      ];
      listenPort = 51821;
      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
      peers = [(peerBase // {
        endpoint = "[2a09:a441::f00f]:51821";
        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
      })];
      postSetup = ''
        wg set wg-tuvok-ee fwmark 0xdead
      '';
    };
    interfaces.wg-tuvok-gnet = ifBase // {
      ips = [
        "2a09:a442::3:1/112"
        "92.118.30.4/31"
      ];
      listenPort = 51822;
      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
      peers = [(peerBase // {
        endpoint = "92.118.28.252:51822";
        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
      })];
      postSetup = ''
        wg set wg-tuvok-gnet fwmark 0xcafe
      '';
    };
  };

  services.unifi = {
    enable = true;
    openFirewall = false;
    unifiPackage = depot.pkgs.unifi;
  };
  services.prometheus.exporters.unifi-poller = {
    enable = true;
    controllers = [{
      url = "https://localhost:8443";
      verify_ssl = false;
      user = "unifipoller";
      pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; };
    }];
  };

  networking.firewall = {
    interfaces.en-general = {
      allowedTCPPorts = [
        8080 6789  # Unifi
        53  # DNS
      ];
      allowedUDPPorts = [
        3478 10001  # Unifi
        53  # DNS
      ];
    };
    interfaces.vl-eduroam = {
      allowedTCPPorts = [
        53  # DNS
      ];
      allowedUDPPorts = [
        53  # DNS
      ];
    };
    interfaces.en-virginmedia = {
      allowedUDPPorts = [
        51820
      ];
    };
    interfaces.en-ee = {
      allowedUDPPorts = [
        51821
      ];
    };
    interfaces.en-gnet = {
      allowedUDPPorts = [
        51822
      ];
    };
    interfaces.wg-tuvok-ee = {
      allowedUDPPorts = [
        3784  # BFD
      ];
    };
    interfaces.wg-tuvok-vm = {
      allowedUDPPorts = [
        3784  # BFD
      ];
    };
    interfaces.wg-tuvok-gnet = {
      allowedUDPPorts = [
        3784  # BFD
      ];
    };
    extraCommands = ''
      ip46tables -F FORWARD

      ip46tables -N ts-forward || true
      ip46tables -A FORWARD -j ts-forward

      iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
      ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360

      ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT
      ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-vm -j ACCEPT
      ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT
      ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT
    '';
  };

  environment.systemPackages = with pkgs; [
    ethtool
  ];

  services.coredns = {
    enable = true;
    config = ''
      .:53 {
        bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.253 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1
        acl {
          allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24
          block
        }
        hosts /dev/null {
          #216.239.38.120 stadia.google.com stadia.com
          fallthrough
        }
        loadbalance
        forward . tls://8.8.8.8 tls://8.8.4.4 {
          tls_servername dns.google
        }
        cache {
          success 4096
          denial 1024
          prefetch 512
        }
        prometheus :9153
        errors
        log
      }
    '';
  };
  my.prometheus.additionalExporterPorts.coredns = 9153;
  networking.resolvconf.extraConfig = ''
    name_servers='127.0.0.253'
  '';

  services.prometheus.exporters.smokeping = {
    enable = true;
    hosts = [
      "8.8.8.8"  # Google Public DNS
      "2001:4860:4860::8888"
      "youtube.com" "ads.google.com" "google.com"

      "1.1.1.1"  # Cloudflare DNS
      "2606:4700:4700::1111"

      "twitter.com"

      "store.steampowered.com"
      "api.steampowered.com"

      "euw1.api.riotgames.com"  # League of Legends EUW

      "eu.battle.net"
      "185.60.112.157" "185.60.112.158"  # Diablo 3/HotS/Hearthstone
      "185.60.114.159"  # Overwatch

      # BFOB TS
      "2a01:a500:85:3::2"
      "37.9.61.53"

      # lukegb01.ring.nlnog.net
      "2a09:a441::13"
      "92.118.28.13"
    ];
  };

  services.bird2 = {
    enable = true;
    config = ''
      router id 92.118.30.254;

      protocol kernel {
        kernel table 150;
        metric 0;
        ipv4 {
          import none;
          export all;
        };
      };
      protocol kernel {
        kernel table 150;
        metric 0;
        ipv6 {
          import none;
          export all;
        };
      };
      protocol device {};

      protocol static export4 {
        ipv4 {};
        route 0.0.0.0/0 via 92.118.30.1 bfd {
          # Virgin Media
          preference = 100;
        };
        route 0.0.0.0/0 via 92.118.30.3 bfd {
          # EE
          preference = 10;
        };
        route 0.0.0.0/0 via 92.118.30.5 bfd {
          # GNetwork
          preference = 200;
        };
      };
      protocol static export6 {
        ipv6 {};
        route ::/0 via 2a09:a442::1:2 bfd {
          # Virgin Media
          preference = 100;
          krt_prefsrc = 2a09:a443::1;
        };
        route ::/0 via 2a09:a442::2:2 bfd {
          # EE
          preference = 10;
          krt_prefsrc = 2a09:a443::1;
        };
        route ::/0 via 2a09:a442::3:2 bfd {
          # GNetwork
          preference = 200;
          krt_prefsrc = 2a09:a443::1;
        };

        # Covering route...
        route 2a09:a443::/64 via "en-general";
        route 2a09:a443:1::/48 via "en-general";
        route 2a09:a443:2::/64 via "vl-eduroam";
        route 2a09:a443:3::/48 via "vl-eduroam";
        route 2a09:a443::/32 unreachable;
      };

      protocol bfd {
        interface "*" {
          min rx interval 10ms;
          min tx interval 50ms;
          idle tx interval 1s;
          multiplier 20;
        };
        neighbor 92.118.30.1;
        neighbor 2a09:a442::1:2;
        neighbor 92.118.30.3;
        neighbor 2a09:a442::2:2;
        neighbor 92.118.30.5;
        neighbor 2a09:a442::3:2;
      };
    '';
  };

  services.radvd = {
    enable = true;
    config = ''
      interface en-general {
        AdvSendAdvert on;
        AdvLinkMTU 1420;  # Wireguard
        AdvManagedFlag on;

        RDNSS 2a09:a443::1 {};
        DNSSL house.as205479.net {};

        prefix 2a09:a443::/64 {
          AdvOnLink on;
          AdvAutonomous on;
        };
        prefix 2a09:a443:1::/48 {
          AdvOnLink on;
          AdvAutonomous off;
        };
      };
      interface vl-eduroam {
        AdvSendAdvert on;
        AdvLinkMTU 1420;  # Wireguard
        AdvManagedFlag on;

        RDNSS 2a09:a443:2::1 {};
        DNSSL eduroam.as205479.net {};

        prefix 2a09:a443:2::/64 {
          AdvOnLink on;
          AdvAutonomous on;
        };
        prefix 2a09:a443:3::/48 {
          AdvOnLink on;
          AdvAutonomous off;
        };
      };
    '';
  };
  services.dhcpd6 = {
    enable = true;
    interfaces = ["en-general" "vl-eduroam"];
    authoritative = true;
    extraConfig = ''
      subnet6 2a09:a443:1::/48 {
        range6 2a09:a443:1:1::/64;
        range6 2a09:a443:1:2::/64 temporary;
        prefix6 2a09:a443:1:1000:: 2a09:a443:1:ff00:: /56;

        option dhcp6.name-servers 2a09:a443:1::1;
        option dhcp6.domain-search "house.as205479.net";
      }
      subnet6 2a09:a443:3::/48 {
        range6 2a09:a443:3:1::/64;
        range6 2a09:a443:3:2::/64 temporary;
        prefix6 2a09:a443:3:1000:: 2a09:a443:3:ff00:: /56;

        option dhcp6.name-servers 2a09:a443:3::1;
        option dhcp6.domain-search "eduroam.as205479.net";
      }
    '';
  };

  systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
    ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
      -web.listen-address 0.0.0.0:9324 \
      -bird.socket /var/run/bird.ctl \
      -bird.v2=true \
      -format.new=true
  '';

  systemd.services.ee-scrape-data = let
    scriptFile = ./ee-scrape-data.py;
    python = pkgs.python3.withPackages (pm: with pm; [
      requests
      beautifulsoup4
      html5lib
    ]);
  in {
    enable = true;
    serviceConfig = {
      Type = "oneshot";
      ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
    };
  };
  systemd.timers.ee-scrape-data = {
    enable = true;
    wantedBy = [ "multi-user.target" ];
    timerConfig = {
      OnBootSec = "2m";
      OnUnitInactiveSec = "1m";
      RandomizedDelaySec = "20";
    };
  };

  system.stateVersion = "21.03";
}