# SPDX-FileCopyrightText: 2020 Luke Granger-Brown # # SPDX-License-Identifier: Apache-2.0 { depot, lib, pkgs, rebuilder, config, ... }: let inherit (depot.ops) secrets; in { imports = [ # We include this just so it sets some sysctls and firewall settings. ../lib/bgp.nix ]; boot.initrd.availableKernelModules = [ "sd_mod" "ahci" "usb_storage" "usbhid" ]; boot.kernelParams = [ "mitigations=off" ]; fileSystems = { "/" = { device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11"; fsType = "ext4"; }; "/boot" = { device = "/dev/disk/by-uuid/AB36-5BE4"; fsType = "vfat"; }; }; nix.maxJobs = lib.mkDefault 4; # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; # Networking! networking = { # Routing tables: # bgp (150) -- contains default routes over WG tunnels # ee (201) -- table contains a static default route via EE # main (254) -- DHCP routes (aka VM) # Conventional lookup order is bgp, main. ee has routing rules. hostName = "swann"; # Define your hostname. domain = "int.as205479.net"; nameservers = ["8.8.8.8" "8.8.4.4"]; useDHCP = false; interfaces = { lo = { ipv4.addresses = [ { address = "127.0.0.1"; prefixLength = 8; } { address = "92.118.30.254"; prefixLength = 32; } { address = "92.118.30.253"; prefixLength = 32; } ]; }; en-virginmedia = { useDHCP = true; macAddress = "e4:3a:6e:16:07:61"; }; en-ee = { ipv4.addresses = [ { address = "192.168.200.2"; prefixLength = 24; } ]; ipv4.routes = [{ via = "192.168.200.1"; address = "0.0.0.0"; prefixLength = 0; options.table = "201"; }]; }; en-general = { ipv4.addresses = [ { address = "192.168.1.1"; prefixLength = 23; } { address = "92.118.30.17"; prefixLength = 28; } ]; ipv6.addresses = [ { address = "2a09:a443::1"; prefixLength = 64; } { address = "2a09:a443:1::1"; prefixLength = 48; } ]; }; vl-eduroam = { ipv4.addresses = [ { address = "192.168.10.1"; prefixLength = 24; } ]; ipv6.addresses = [ { address = "2a09:a443:2::1"; prefixLength = 64; } { address = "2a09:a443:3::1"; prefixLength = 48; } ]; }; }; vlans = { vl-eduroam = { id = 100; interface = "en-general"; }; }; dhcpcd.extraConfig = '' interface en-virginmedia metric 100 interface en-ee metric 250 ''; localCommands = let claimedPriorities = { min = 10000; max = 10100; }; rules = [ # Route traffic to EE via WG... via EE. { priority = 10000; both = "fwmark 0xdead table 201"; } # Route traffic to VM via WG... via VM DHCP in main table. { priority = 10001; both = "fwmark 0xbeef table main"; } # Make ping work over the tunnels. { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; } { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; } # Now some subset of RFC1918 via main table too. { priority = 10020; v4 = "to 192.168.0.0/16 table main"; } { priority = 10021; v4 = "to 10.0.0.0/8 table main"; } { priority = 10022; v4 = "to 172.16.0.0/12 table main"; } # And the linknets... { priority = 10023; v4 = "to 92.118.30.0/24 table main"; } { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; } { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; } # And the Google VIP I'm (ab)using for Stadia (see CoreDNS below). { priority = 10030; v4 = "to 216.239.38.120/32 table main"; } # add-on.ee.co.uk goes via EE. { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; } # Anything originating from 192.168.200.0/24 should go via EE too. { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; } # Everything else over WG. { priority = 10099; both = "table 150"; } ]; clearRules = map (x: '' ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true '') (lib.range claimedPriorities.min (claimedPriorities.max - 1)); ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }: assert (both == "" || (v4 == "" && v6 == "")); assert priority >= claimedPriorities.min; assert priority < claimedPriorities.max; let rv4 = if v4 != "" then v4 else both; rv6 = if v6 != "" then v6 else both; in '' ${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""} ${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""} ''; addRules = map ruleToLine rules; in '' ${lib.concatStringsSep "\n" clearRules} ${lib.concatStringsSep "\n" addRules} ip -4 route flush table 151 >/dev/null 2>&1 || true ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151 ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151 ip -6 route flush table 151 >/dev/null 2>&1 || true ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151 ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151 ip -4 route flush table 152 >/dev/null 2>&1 || true ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151 ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152 ip -6 route flush table 152 >/dev/null 2>&1 || true ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152 ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152 ''; }; my.ip.tailscale = "100.102.224.95"; services.udev.extraRules = '' ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia" ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee" ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general" ''; boot.kernel.sysctl = { "net.ipv4.ip_forward" = "1"; "net.ipv6.conf.default.forwarding" = "1"; "net.ipv6.conf.all.forwarding" = "1"; "net.ipv6.conf.en-virginmedia.accept_ra" = "2"; "net.ipv6.conf.en-ee.accept_ra" = "2"; }; networking.nat = { enable = true; internalInterfaces = ["en-general"]; externalInterface = "en-virginmedia"; extraCommands = '' # Send PS5 RTMP to totoro instead. # See DHCP static lease. iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40 # NAT packets going over EE plain. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE # SNAT packets we're sending over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254 iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254 # eduroam # > mark incoming eduroam packets iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2 # > NAT packets going over EE/VM. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-virginmedia -j MASQUERADE iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE # > NAT packets going over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.253 iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253 ''; }; services.dhcpd4 = { enable = true; interfaces = ["en-general" "vl-eduroam"]; authoritative = true; extraConfig = '' shared-network int { default-lease-time 600; max-lease-time 3600; option interface-mtu 1420; # Wireguard subnet 192.168.1.0 netmask 255.255.255.0 { option subnet-mask 255.255.255.0; option routers 192.168.1.1; option domain-name-servers 192.168.1.1; option domain-name "house.as205479.net"; range 192.168.1.100 192.168.1.200; } subnet 92.118.30.16 netmask 255.255.255.240 { option subnet-mask 255.255.255.240; option routers 92.118.30.17; option domain-name-servers 92.118.30.17; option domain-name "house-ext.as205479.net"; } } subnet 192.168.10.0 netmask 255.255.255.0 { option subnet-mask 255.255.255.0; option routers 192.168.10.1; option domain-name-servers 192.168.10.1; option domain-name "eduroam.as205479.net"; default-lease-time 600; max-lease-time 3600; option interface-mtu 1420; # Wireguard range 192.168.10.100 192.168.10.200; } ''; machines = [ { hostName = "totoro"; ethernetAddress = "40:8d:5c:1f:e8:68"; ipAddress = "192.168.1.40"; } { hostName = "totoro-pfsense"; ethernetAddress = "52:54:00:cf:cd:94"; ipAddress = "192.168.1.41"; } { hostName = "kvm"; ethernetAddress = "00:0d:5d:1b:14:ba"; ipAddress = "192.168.1.50"; } { hostName = "printer-xerox"; ethernetAddress = "9c:93:4e:ad:1f:7b"; ipAddress = "192.168.1.51"; } { hostName = "ps5"; ethernetAddress = "bc:33:29:26:01:5c"; # This is used for DNAT on RTMP, above. ipAddress = "92.118.30.18"; } ]; }; networking.wireguard = let ifBase = { listenPort = null; allowedIPsAsRoutes = false; }; peerBase = { allowedIPs = [ "0.0.0.0/0" "::/0" ]; }; in { enable = true; interfaces.wg-tuvok-vm = ifBase // { ips = [ "2a09:a442::1:1/112" "92.118.30.0/31" ]; listenPort = 51820; privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; peers = [(peerBase // { endpoint = "92.118.28.252:51820"; publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; })]; postSetup = '' wg set wg-tuvok-vm fwmark 0xbeef ''; }; interfaces.wg-tuvok-ee = ifBase // { ips = [ "2a09:a442::2:1/112" "92.118.30.2/31" ]; listenPort = 51821; privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; peers = [(peerBase // { endpoint = "[2a09:a441::f00f]:51821"; publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; })]; postSetup = '' wg set wg-tuvok-ee fwmark 0xdead ''; }; }; services.unifi = { enable = true; openFirewall = false; unifiPackage = depot.pkgs.unifi; }; services.prometheus.exporters.unifi-poller = { enable = true; controllers = [{ url = "https://localhost:8443"; verify_ssl = false; user = "unifipoller"; pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; }; }]; }; networking.firewall = { interfaces.en-general = { allowedTCPPorts = [ 8080 6789 # Unifi 53 # DNS ]; allowedUDPPorts = [ 3478 10001 # Unifi 53 # DNS ]; }; interfaces.vl-eduroam = { allowedTCPPorts = [ 53 # DNS ]; allowedUDPPorts = [ 53 # DNS ]; }; interfaces.wg-tuvok-ee = { allowedUDPPorts = [ 3784 # BFD ]; }; interfaces.wg-tuvok-vm = { allowedUDPPorts = [ 3784 # BFD ]; }; extraCommands = '' ip46tables -F FORWARD ip46tables -N ts-forward || true ip46tables -A FORWARD -j ts-forward iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-vm -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT ''; }; services.ddclient = { enable = false; protocol = "cloudflare"; domains = ["home.lukegb.com"]; zone = "lukegb.com"; passwordFile = pkgs.writeText "cloudflare-token" secrets.cloudflareCredentials.token; use = "if"; extraConfig = '' if=en-virginmedia daemon=0 ''; }; systemd.services.ddclient.serviceConfig.ExecStart = let ddclient = pkgs.perlPackages.buildPerlPackage rec { pname = "ddclient"; version = "3.9.1"; src = pkgs.fetchFromGitHub { owner = "ddclient"; repo = "ddclient"; rev = "11a583b003920f8e15591813598b70061d1a4654"; sha256 = "sha256:1xz09vkii3mc2jmfwx9is07i06iiryv51571vdnl4m5mdnvsmlwb"; }; outputs = [ "out" ]; doCheck = false; buildInputs = with pkgs.perlPackages; [ IOSocketSSL DigestSHA1 DataValidateIP JSONPP ]; nativeBuildInputs = with pkgs; [ autoreconfHook makeWrapper ]; preConfigure = '' touch Makefile.PL ''; postInstall = '' patchShebangs $out/bin/ddclient wrapProgram $out/bin/ddclient \ --suffix PATH : ${lib.makeBinPath (with pkgs; [ pkgs.iproute ])} \ --prefix PERL5LIB : $PERL5LIB ''; }; RuntimeDirectory = "ddclient"; in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf"; environment.systemPackages = with pkgs; [ ethtool ]; services.coredns = { enable = true; config = '' .:53 { bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.53 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1 acl { allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24 block } hosts /dev/null { #216.239.38.120 stadia.google.com stadia.com fallthrough } loadbalance forward . tls://8.8.8.8 tls://8.8.4.4 { tls_servername dns.google } cache { success 4096 denial 1024 prefetch 512 } prometheus :9153 errors log } ''; }; my.prometheus.additionalExporterPorts.coredns = 9153; networking.resolvconf.extraConfig = '' name_servers='127.0.0.53' ''; services.prometheus.exporters.smokeping = { enable = true; hosts = [ "8.8.8.8" # Google Public DNS "2001:4860:4860::8888" "youtube.com" "ads.google.com" "google.com" "1.1.1.1" # Cloudflare DNS "2606:4700:4700::1111" "twitter.com" "store.steampowered.com" "api.steampowered.com" "prod.euw1.lol.riotgames.com" # League of Legends EUW "eu.battle.net" "185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone "185.60.114.159" # Overwatch # BFOB TS "2a01:a500:85:3::2" "37.9.61.53" ]; }; services.bird2 = { enable = true; config = '' router id 92.118.30.254; protocol kernel { kernel table 150; metric 0; ipv4 { import none; export all; }; }; protocol kernel { kernel table 150; metric 0; ipv6 { import none; export all; }; }; protocol device {}; protocol static export4 { ipv4 {}; route 0.0.0.0/0 via 92.118.30.1 bfd { # Virgin Media preference = 100; }; route 0.0.0.0/0 via 92.118.30.3 bfd { # EE preference = 10; }; }; protocol static export6 { ipv6 {}; route ::/0 via 2a09:a442::1:2 bfd { # Virgin Media preference = 100; }; route ::/0 via 2a09:a442::2:2 bfd { # EE preference = 10; }; # Covering route... route 2a09:a443::/64 via "en-general"; route 2a09:a443:1::/48 via "en-general"; route 2a09:a443:2::/64 via "vl-eduroam"; route 2a09:a443:3::/48 via "vl-eduroam"; route 2a09:a443::/32 unreachable; }; protocol bfd { interface "*" { min rx interval 10ms; min tx interval 50ms; idle tx interval 1s; multiplier 20; }; neighbor 92.118.30.1; neighbor 2a09:a442::1:2; neighbor 92.118.30.3; neighbor 2a09:a442::2:2; }; ''; }; services.radvd = { enable = true; config = '' interface en-general { AdvSendAdvert on; AdvLinkMTU 1420; # Wireguard AdvManagedFlag on; RDNSS 2a09:a443::1 {}; DNSSL house.as205479.net {}; prefix 2a09:a443::/64 { AdvOnLink on; AdvAutonomous on; }; prefix 2a09:a443:1::/48 { AdvOnLink on; AdvAutonomous off; }; }; interface vl-eduroam { AdvSendAdvert on; AdvLinkMTU 1420; # Wireguard AdvManagedFlag on; RDNSS 2a09:a443:2::1 {}; DNSSL eduroam.as205479.net {}; prefix 2a09:a443:2::/64 { AdvOnLink on; AdvAutonomous on; }; prefix 2a09:a443:3::/48 { AdvOnLink on; AdvAutonomous off; }; }; ''; }; services.dhcpd6 = { enable = true; interfaces = ["en-general" "vl-eduroam"]; authoritative = true; extraConfig = '' subnet6 2a09:a443:1::/48 { range6 2a09:a443:1:1::/64; range6 2a09:a443:1:2::/64 temporary; prefix6 2a09:a443:1:1000:: 2a09:a443:1:ff00:: /56; option dhcp6.name-servers 2a09:a443:1::1; option dhcp6.domain-search "house.as205479.net"; } subnet6 2a09:a443:3::/48 { range6 2a09:a443:3:1::/64; range6 2a09:a443:3:2::/64 temporary; prefix6 2a09:a443:3:1000:: 2a09:a443:3:ff00:: /56; option dhcp6.name-servers 2a09:a443:3::1; option dhcp6.domain-search "eduroam.as205479.net"; } ''; }; systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce '' ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \ -web.listen-address 0.0.0.0:9324 \ -bird.socket /var/run/bird.ctl \ -bird.v2=true \ -format.new=true ''; systemd.services.ee-scrape-data = let scriptFile = ./ee-scrape-data.py; python = pkgs.python3.withPackages (pm: with pm; [ requests beautifulsoup4 html5lib ]); in { enable = true; serviceConfig = { Type = "oneshot"; ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom"; }; }; systemd.timers.ee-scrape-data = { enable = true; wantedBy = [ "multi-user.target" ]; timerConfig = { OnBootSec = "2m"; OnUnitInactiveSec = "1m"; RandomizedDelaySec = "20"; }; }; system.stateVersion = "21.03"; }