# SPDX-FileCopyrightText: 2020 Luke Granger-Brown # # SPDX-License-Identifier: Apache-2.0 { depot, lib, pkgs, config, ... }: let inherit (depot.ops) secrets; inherit (lib) mkMerge mkForce; in { imports = [ # We include this just so it sets some sysctls and firewall settings. ../lib/bgp.nix ]; config = mkMerge [ { boot.initrd.availableKernelModules = [ "sd_mod" "ahci" "usb_storage" "usbhid" ]; boot.kernelParams = [ "mitigations=off" ]; fileSystems = { "/" = { device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11"; fsType = "ext4"; }; "/boot" = { device = "/dev/disk/by-uuid/AB36-5BE4"; fsType = "vfat"; }; }; nix.settings.max-jobs = lib.mkDefault 4; # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; # Networking! networking = { # Routing tables: # bgp (150) -- contains default routes over WG tunnels # wg-ee (152) -- contains default routes over WG tunnels # wg-gnet (153) -- contains default routes over WG tunnels # ee (201) -- table contains a default route via EE # vm (202) -- table contains a default route via VM # gnet (203) -- table contains a default route via gnetwork # main (254) -- basically empty hostName = "swann"; # Define your hostname. domain = "int.as205479.net"; nameservers = ["8.8.8.8" "8.8.4.4"]; useNetworkd = true; interfaces = { lo = { ipv4.addresses = [ { address = "127.0.0.1"; prefixLength = 8; } { address = "92.118.30.254"; prefixLength = 32; } { address = "92.118.30.253"; prefixLength = 32; } ]; }; en-gnet = { useDHCP = true; ipv4.addresses = [ { address = "192.168.201.2"; prefixLength = 24; } ]; # Additional options configured in networkd. }; en-ee = { useDHCP = true; ipv4.addresses = [ { address = "192.168.200.2"; prefixLength = 24; } ]; # Additional options configured in networkd. }; br-internal = { ipv4.addresses = [ { address = "192.168.1.1"; prefixLength = 23; } { address = "92.118.30.17"; prefixLength = 28; } ]; ipv6.addresses = [ { address = "2a09:a443::1"; prefixLength = 64; } { address = "2a09:a443:1::1"; prefixLength = 48; } ]; }; vl-eduroam = { ipv4.addresses = [ { address = "192.168.10.1"; prefixLength = 24; } ]; ipv6.addresses = [ { address = "2a09:a443:2::1"; prefixLength = 64; } { address = "2a09:a443:3::1"; prefixLength = 48; } ]; }; }; }; systemd.network = let hexToInt = h: (builtins.fromTOML "h = ${h}").h; physicalNetwork = rtID: wireguardFwmark: extraRules: { dhcpV4Config.RouteTable = rtID; ipv6AcceptRAConfig.RouteTable = rtID; routingPolicyRules = [{ routingPolicyRuleConfig = { Family = "both"; FirewallMark = hexToInt wireguardFwmark; Priority = 10000; Table = rtID; }; }] ++ extraRules; }; wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: { matchConfig.Name = linkName; routes = let replaceV4Octet = v4: fn: let pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4; in "${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}"; replaceV6Octet = v6: fn: let pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6; in "${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}"; in [ { routeConfig = { Destination = "${v4Linknet}/31"; Table = rtID; }; } { routeConfig = { Gateway = replaceV4Octet v4Linknet (n: n + 1); Table = rtID; }; } { routeConfig = { Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112"; Table = rtID; }; } { routeConfig = { Gateway = replaceV6Octet v6Linknet (n: n + 1); Table = rtID; }; } ]; networkConfig = { Address = [ "${v4Linknet}/31" "${v6Linknet}/112" ]; }; routingPolicyRules = [ (tailscaleRule (relativePriority + 5000) rtID) # Allow picking destination by source IP. { routingPolicyRuleConfig = { Family = "ipv4"; From = v4Linknet; Priority = 10010; Table = rtID; }; } { routingPolicyRuleConfig = { Family = "ipv6"; From = v6Linknet; Priority = 10010; Table = rtID; }; } { # Catch-all mop-up rule at the end. routingPolicyRuleConfig = { Family = "both"; Priority = relativePriority + 10090; Table = rtID; }; } ]; }; tailscaleRule = priority: table: { # Route Tailscale (fwmark 0x80000) via Wireguard first. routingPolicyRuleConfig = { Family = "both"; FirewallMark = hexToInt "0x80000"; Priority = priority; Table = table; }; }; in let routeTables = { bgp = 150; wg-ee = 152; wg-gnet = 153; ee = 201; gnet = 203; }; in { enable = true; config.routeTables = routeTables; networks."50-wg-tuvok-ee" = wireguardNetwork { linkName = "wg-tuvok-ee"; relativePriority = 3; rtID = routeTables.wg-ee; v4Linknet = "92.118.30.2"; v6Linknet = "2a09:a442::2:1"; }; networks."50-wg-tuvok-gnet" = wireguardNetwork { linkName = "wg-tuvok-gnet"; relativePriority = 1; rtID = routeTables.wg-gnet; v4Linknet = "92.118.30.4"; v6Linknet = "2a09:a442::3:1"; }; networks."40-lo" = { routingPolicyRules = let viaMain = priority: to: { routingPolicyRuleConfig = { To = to; Table = "main"; Priority = priority; }; }; blackhole = fwmark: { routingPolicyRuleConfig = { Family = "both"; FirewallMark = hexToInt fwmark; Priority = 10001; Type = "unreachable"; }; }; in [ (tailscaleRule 5000 150) # Blackhole connections that should be routed over individual interfaces. (blackhole "0xdead") (blackhole "0xbeef") (blackhole "0xcafe") # RFC 1918 via main table. (viaMain 10020 "192.168.0.0/16") (viaMain 10021 "10.0.0.0/8") (viaMain 10022 "172.16.0.0/12") # and the linknets. (viaMain 10023 "92.118.30.0/24") (viaMain 10024 "2a09:a442::1:0/112") (viaMain 10025 "2a09:a442::2:0/112") (viaMain 10026 "2a09:a442::3:0/112") { # Catch-all "go via WG" routingPolicyRuleConfig = { Family = "both"; Priority = 10080; Table = routeTables.bgp; }; } ]; }; networks."40-en-ee" = (physicalNetwork routeTables.ee "0xdead" [{ routingPolicyRuleConfig = { # add-on.ee.co.uk goes via EE. To = "82.192.97.153/32"; Table = routeTables.ee; Priority = 10031; }; } { routingPolicyRuleConfig = { # as does anything from 192.168.200.0/24. From = "192.168.200.0/24"; Table = routeTables.ee; Priority = 10031; }; }]) // { linkConfig.RequiredForOnline = "no"; }; networks."40-en-gnet" = (physicalNetwork routeTables.gnet "0xcafe" []); networks."40-br-internal" = { networkConfig.VLAN = [ "vl-eduroam" ]; }; networks."40-en-int-eth" = { matchConfig.Name = "en-int-eth"; networkConfig.Bridge = "br-internal"; }; networks."40-en-int-sfp" = { matchConfig.Name = "en-int-sfp"; networkConfig.Bridge = "br-internal"; }; netdevs = let wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: { netdevConfig = { Name = name; Kind = "wireguard"; Description = "WireGuard tunnel ${name}"; }; wireguardConfig = { ListenPort = listenPort; PrivateKeyFile = pkgs.writeText "${name}" privateKey; # TODO: PrivateKeyFile FirewallMark = hexToInt fwmark; RouteTable = "off"; }; wireguardPeers = [{ wireguardPeerConfig = { Endpoint = endpoint; PublicKey = publicKey; AllowedIPs = [ "0.0.0.0/0" "::/0" ]; }; }]; }; tuvokWireguard = args: wireguard (args // { privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; }); in { "40-wg-tuvok-ee" = tuvokWireguard { name = "wg-tuvok-ee"; listenPort = 51821; endpoint = "[2a09:a441::f00f]:51821"; fwmark = "0xdead"; }; "40-wg-tuvok-gnet" = tuvokWireguard { name = "wg-tuvok-gnet"; listenPort = 51822; endpoint = "92.118.28.252:51822"; fwmark = "0xcafe"; }; "20-br-internal" = { netdevConfig = { Name = "br-internal"; Kind = "bridge"; Description = "Bridge br-internal"; }; extraConfig = '' [Bridge] VLANFiltering=true MulticastQuerier=true MulticastSnooping=true STP=true VLANProtocol=802.1q MulticastIGMPVersion=3 ''; }; "25-vl-eduroam" = { netdevConfig = { Name = "vl-eduroam"; Kind = "vlan"; Description = "Eduroam VLAN on br-internal"; }; vlanConfig = { Id = 100; }; }; }; }; services.mstpd.enable = true; my.ip.tailscale = "100.102.224.95"; services.udev.extraRules = '' ATTR{address}=="e4:3a:6e:16:07:63", DRIVERS=="?*", NAME="en-ee" ATTR{address}=="e4:3a:6e:16:07:64", DRIVERS=="?*", NAME="en-gnet" ATTR{address}=="e4:3a:6e:16:07:67", DRIVERS=="?*", NAME="en-int-eth" ATTR{address}=="e4:3a:6e:16:08:bc", DRIVERS=="?*", NAME="en-int-sfp" ''; boot.kernel.sysctl = { "net.ipv4.ip_forward" = "1"; "net.ipv6.conf.default.forwarding" = "1"; "net.ipv6.conf.all.forwarding" = "1"; "net.ipv6.conf.en-ee.accept_ra" = "2"; "net.ipv6.conf.en-gnet.accept_ra" = "2"; }; networking.nat = { enable = true; internalInterfaces = ["br-internal"]; externalInterface = "en-gnet"; extraCommands = '' # Send PS5 RTMP to totoro instead. # See DHCP static lease. iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40 # NAT packets going over EE plain. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE # NAT packets going over GNetwork plain. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE # SNAT packets we're sending over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254 iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254 # eduroam # > mark incoming eduroam packets iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2 # > NAT packets going out directly. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE # > NAT packets going over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253 iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253 ''; }; services.dhcpd4 = { enable = true; interfaces = ["br-internal" "vl-eduroam"]; authoritative = true; extraConfig = '' shared-network int { default-lease-time 3600; max-lease-time 86400; option interface-mtu 1420; # Wireguard subnet 192.168.1.0 netmask 255.255.255.0 { option subnet-mask 255.255.255.0; option routers 192.168.1.1; option domain-name-servers 192.168.1.1; option domain-name "house.as205479.net"; range 192.168.1.100 192.168.1.200; } subnet 92.118.30.16 netmask 255.255.255.240 { option subnet-mask 255.255.255.240; option routers 92.118.30.17; option domain-name-servers 92.118.30.17; option domain-name "house-ext.as205479.net"; } } subnet 192.168.10.0 netmask 255.255.255.0 { option subnet-mask 255.255.255.0; option routers 192.168.10.1; option domain-name-servers 192.168.10.1; option domain-name "eduroam.as205479.net"; default-lease-time 600; max-lease-time 3600; option interface-mtu 1420; # Wireguard range 192.168.10.100 192.168.10.200; } ''; machines = [ { hostName = "totoro"; ethernetAddress = "40:8d:5c:1f:e8:68"; ipAddress = "192.168.1.40"; } { hostName = "totoro-pfsense"; ethernetAddress = "52:54:00:cf:cd:94"; ipAddress = "192.168.1.41"; } { hostName = "kvm"; ethernetAddress = "00:0d:5d:1b:14:ba"; ipAddress = "192.168.1.50"; } { hostName = "printer-xerox"; ethernetAddress = "9c:93:4e:ad:1f:7b"; ipAddress = "192.168.1.51"; } { hostName = "ps5"; ethernetAddress = "bc:33:29:26:01:5c"; # This is used for DNAT on RTMP, above. ipAddress = "92.118.30.18"; } ]; }; systemd.services.dhcpd4 = { wants = [ "systemd-networkd-wait-online.service" ]; after = [ "systemd-networkd-wait-online.service" ]; }; services.unifi = { enable = true; openFirewall = false; unifiPackage = depot.pkgs.unifi; }; services.prometheus.exporters.unifi-poller = { enable = true; controllers = [{ url = "https://localhost:8443"; verify_ssl = false; user = "unifipoller"; pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; }; }]; }; networking.firewall = { interfaces.br-internal = { allowedTCPPorts = [ 8080 6789 # Unifi 53 # DNS ]; allowedUDPPorts = [ 3478 10001 # Unifi 53 # DNS ]; }; interfaces.vl-eduroam = { allowedTCPPorts = [ 53 # DNS ]; allowedUDPPorts = [ 53 # DNS ]; }; interfaces.en-ee = { allowedUDPPorts = [ 51821 ]; }; interfaces.en-gnet = { allowedUDPPorts = [ 51822 ]; }; interfaces.wg-tuvok-ee = { allowedUDPPorts = [ 3784 # BFD ]; }; interfaces.wg-tuvok-gnet = { allowedUDPPorts = [ 3784 # BFD ]; }; extraCommands = '' ip46tables -F FORWARD ip46tables -N ts-forward || true ip46tables -A FORWARD -j ts-forward iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT ''; }; environment.systemPackages = with pkgs; [ ethtool (writeShellApplication { name = "bridge-stp"; runtimeInputs = [ mstpd ]; text = '' BRIDGES=("br-internal") for BRIDGE in "''${BRIDGES[@]}"; do if [[ "$BRIDGE" = "$1" ]]; then if [[ "$2" = "start" ]]; then mstpctl addbridge "$BRIDGE" exit 0 elif [[ "$2" = "stop" ]]; then mstpctl delbridge "$BRIDGE" exit 0 fi exit 1 fi done exit 1 ''; }) ]; services.coredns = { enable = true; config = '' .:53 { bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.253 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1 acl { allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24 block } hosts /dev/null { #216.239.38.120 stadia.google.com stadia.com fallthrough } loadbalance forward . tls://8.8.8.8 tls://8.8.4.4 { tls_servername dns.google } cache { success 4096 denial 1024 prefetch 512 } prometheus :9153 errors log } ''; }; systemd.services.coredns = { wants = [ "systemd-networkd-wait-online.service" ]; after = [ "systemd-networkd-wait-online.service" ]; }; my.prometheus.additionalExporterPorts.coredns = 9153; networking.resolvconf.extraConfig = '' name_servers='127.0.0.253' ''; services.prometheus.exporters.smokeping = { enable = true; hosts = [ "8.8.8.8" # Google Public DNS "2001:4860:4860::8888" "youtube.com" "ads.google.com" "google.com" "1.1.1.1" # Cloudflare DNS "2606:4700:4700::1111" "twitter.com" "store.steampowered.com" "api.steampowered.com" "euw1.api.riotgames.com" # League of Legends EUW "eu.battle.net" "185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone "185.60.114.159" # Overwatch # BFOB TS "2a01:a500:85:3::2" "37.9.61.53" # lukegb01.ring.nlnog.net "2a09:a441::13" "92.118.28.13" ]; }; services.bird2 = { enable = true; config = '' router id 92.118.30.254; protocol kernel { kernel table 150; metric 0; ipv4 { import none; export all; }; }; protocol kernel { kernel table 150; metric 0; ipv6 { import none; export all; }; }; protocol device {}; protocol static export4 { ipv4 {}; route 0.0.0.0/0 via 92.118.30.1 bfd { # Virgin Media preference = 100; }; route 0.0.0.0/0 via 92.118.30.3 bfd { # EE preference = 10; }; route 0.0.0.0/0 via 92.118.30.5 bfd { # GNetwork preference = 200; }; }; protocol static export6 { ipv6 {}; route ::/0 via 2a09:a442::1:2 bfd { # Virgin Media preference = 100; krt_prefsrc = 2a09:a443::1; }; route ::/0 via 2a09:a442::2:2 bfd { # EE preference = 10; krt_prefsrc = 2a09:a443::1; }; route ::/0 via 2a09:a442::3:2 bfd { # GNetwork preference = 200; krt_prefsrc = 2a09:a443::1; }; # Covering route... route 2a09:a443::/64 via "br-internal"; route 2a09:a443:1::/48 via "br-internal"; route 2a09:a443:2::/64 via "vl-eduroam"; route 2a09:a443:3::/48 via "vl-eduroam"; route 2a09:a443::/32 unreachable; }; protocol bfd { interface "*" { min rx interval 10ms; min tx interval 50ms; idle tx interval 1s; multiplier 20; }; neighbor 92.118.30.1; neighbor 2a09:a442::1:2; neighbor 92.118.30.3; neighbor 2a09:a442::2:2; neighbor 92.118.30.5; neighbor 2a09:a442::3:2; }; ''; }; services.radvd = { enable = true; config = '' interface br-internal { AdvSendAdvert on; AdvLinkMTU 1420; # Wireguard AdvManagedFlag on; RDNSS 2a09:a443::1 {}; DNSSL house.as205479.net {}; prefix 2a09:a443::/64 { AdvOnLink on; AdvAutonomous on; }; prefix 2a09:a443:1::/48 { AdvOnLink on; AdvAutonomous off; }; }; interface vl-eduroam { AdvSendAdvert on; AdvLinkMTU 1420; # Wireguard AdvManagedFlag on; RDNSS 2a09:a443:2::1 {}; DNSSL eduroam.as205479.net {}; prefix 2a09:a443:2::/64 { AdvOnLink on; AdvAutonomous on; }; prefix 2a09:a443:3::/48 { AdvOnLink on; AdvAutonomous off; }; }; ''; }; services.dhcpd6 = { enable = true; interfaces = ["br-internal" "vl-eduroam"]; authoritative = true; extraConfig = '' subnet6 2a09:a443:1::/48 { range6 2a09:a443:1:1::/64; range6 2a09:a443:1:2::/64 temporary; prefix6 2a09:a443:1:1000:: 2a09:a443:1:ff00:: /56; option dhcp6.name-servers 2a09:a443:1::1; option dhcp6.domain-search "house.as205479.net"; } subnet6 2a09:a443:3::/48 { range6 2a09:a443:3:1::/64; range6 2a09:a443:3:2::/64 temporary; prefix6 2a09:a443:3:1000:: 2a09:a443:3:ff00:: /56; option dhcp6.name-servers 2a09:a443:3::1; option dhcp6.domain-search "eduroam.as205479.net"; } ''; }; systemd.services.dhcpd6 = { wants = [ "systemd-networkd-wait-online.service" ]; after = [ "systemd-networkd-wait-online.service" ]; }; systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce '' ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \ -web.listen-address 0.0.0.0:9324 \ -bird.socket /var/run/bird.ctl \ -bird.v2=true \ -format.new=true ''; systemd.services.ee-scrape-data = let scriptFile = ./ee-scrape-data.py; python = pkgs.python3.withPackages (pm: with pm; [ requests beautifulsoup4 html5lib ]); in { enable = true; serviceConfig = { Type = "oneshot"; ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom"; }; }; systemd.timers.ee-scrape-data = { enable = true; wantedBy = [ "multi-user.target" ]; timerConfig = { OnBootSec = "2m"; OnUnitInactiveSec = "1m"; RandomizedDelaySec = "20"; }; }; system.stateVersion = "21.03"; } { # Minimize writes to storage. boot.tmpOnTmpfs = true; services.journald.extraConfig = '' Storage=volatile ''; systemd.services.tailscaled.environment.TS_LOGS_DIR = "/var/run/tailscale"; services.unifi.enable = mkForce false; } ]; }