depot/ops/nixos/swann/default.nix
Luke Granger-Brown 6cfcd10e06 swann: use the router's public IP when making connections
For v6, the link is on an unrouted subnet so there's no way to address it from
outside. We don't want Linux to use the v6 subnet for connections it makes, so
we ask politely that the source on the route is actually an IP address that we
Like.
2022-01-01 02:11:59 +00:00

658 lines
19 KiB
Nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0
{ depot, lib, pkgs, rebuilder, config, ... }:
let
inherit (depot.ops) secrets;
in {
imports = [
# We include this just so it sets some sysctls and firewall settings.
../lib/bgp.nix
];
boot.initrd.availableKernelModules = [
"sd_mod"
"ahci"
"usb_storage"
"usbhid"
];
boot.kernelParams = [ "mitigations=off" ];
fileSystems = {
"/" = {
device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
fsType = "ext4";
};
"/boot" = {
device = "/dev/disk/by-uuid/AB36-5BE4";
fsType = "vfat";
};
};
nix.maxJobs = lib.mkDefault 4;
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
# Networking!
networking = {
# Routing tables:
# bgp (150) -- contains default routes over WG tunnels
# ee (201) -- table contains a static default route via EE
# main (254) -- DHCP routes (aka VM)
# Conventional lookup order is bgp, main. ee has routing rules.
hostName = "swann"; # Define your hostname.
domain = "int.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"];
useDHCP = false;
interfaces = {
lo = {
ipv4.addresses = [
{ address = "127.0.0.1"; prefixLength = 8; }
{ address = "92.118.30.254"; prefixLength = 32; }
{ address = "92.118.30.253"; prefixLength = 32; }
];
};
en-virginmedia = {
useDHCP = true;
macAddress = "e4:3a:6e:16:07:61";
};
en-ee = {
ipv4.addresses = [
{ address = "192.168.200.2"; prefixLength = 24; }
];
ipv4.routes = [{
via = "192.168.200.1";
address = "0.0.0.0";
prefixLength = 0;
options.table = "201";
}];
};
en-general = {
ipv4.addresses = [
{ address = "192.168.1.1"; prefixLength = 23; }
{ address = "92.118.30.17"; prefixLength = 28; }
];
ipv6.addresses = [
{ address = "2a09:a443::1"; prefixLength = 64; }
{ address = "2a09:a443:1::1"; prefixLength = 48; }
];
};
vl-eduroam = {
ipv4.addresses = [
{ address = "192.168.10.1"; prefixLength = 24; }
];
ipv6.addresses = [
{ address = "2a09:a443:2::1"; prefixLength = 64; }
{ address = "2a09:a443:3::1"; prefixLength = 48; }
];
};
};
vlans = {
vl-eduroam = {
id = 100;
interface = "en-general";
};
};
dhcpcd.extraConfig = ''
interface en-virginmedia
metric 100
interface en-ee
metric 250
'';
localCommands = let
claimedPriorities = { min = 10000; max = 10100; };
rules = [
# Route traffic to EE via WG... via EE.
{ priority = 10000; both = "fwmark 0xdead table 201"; }
# Route traffic to VM via WG... via VM DHCP in main table.
{ priority = 10001; both = "fwmark 0xbeef table main"; }
# Make ping work over the tunnels.
{ priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
{ priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
# Now some subset of RFC1918 via main table too.
{ priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
{ priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
{ priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
# And the linknets...
{ priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
{ priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
{ priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
# And the Google VIP I'm (ab)using for Stadia (see CoreDNS below).
{ priority = 10030; v4 = "to 216.239.38.120/32 table main"; }
# add-on.ee.co.uk goes via EE.
{ priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
# Anything originating from 192.168.200.0/24 should go via EE too.
{ priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
# Everything else over WG.
{ priority = 10099; both = "table 150"; }
];
clearRules = map (x: ''
ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
'') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
assert (both == "" || (v4 == "" && v6 == ""));
assert priority >= claimedPriorities.min;
assert priority < claimedPriorities.max;
let
rv4 = if v4 != "" then v4 else both;
rv6 = if v6 != "" then v6 else both;
in ''
${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
'';
addRules = map ruleToLine rules;
in ''
${lib.concatStringsSep "\n" clearRules}
${lib.concatStringsSep "\n" addRules}
ip -4 route flush table 151 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
ip -6 route flush table 151 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151
ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151
ip -4 route flush table 152 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
ip -6 route flush table 152 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152
'';
};
my.ip.tailscale = "100.102.224.95";
services.udev.extraRules = ''
ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
'';
boot.kernel.sysctl = {
"net.ipv4.ip_forward" = "1";
"net.ipv6.conf.default.forwarding" = "1";
"net.ipv6.conf.all.forwarding" = "1";
"net.ipv6.conf.en-virginmedia.accept_ra" = "2";
"net.ipv6.conf.en-ee.accept_ra" = "2";
};
networking.nat = {
enable = true;
internalInterfaces = ["en-general"];
externalInterface = "en-virginmedia";
extraCommands = ''
# Send PS5 RTMP to totoro instead.
# See DHCP static lease.
iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40
# NAT packets going over EE plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
# SNAT packets we're sending over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
# eduroam
# > mark incoming eduroam packets
iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2
# > NAT packets going over EE/VM.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-virginmedia -j MASQUERADE
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE
# > NAT packets going over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.253
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253
'';
};
services.dhcpd4 = {
enable = true;
interfaces = ["en-general" "vl-eduroam"];
authoritative = true;
extraConfig = ''
shared-network int {
default-lease-time 600;
max-lease-time 3600;
option interface-mtu 1420; # Wireguard
subnet 192.168.1.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option routers 192.168.1.1;
option domain-name-servers 192.168.1.1;
option domain-name "house.as205479.net";
range 192.168.1.100 192.168.1.200;
}
subnet 92.118.30.16 netmask 255.255.255.240 {
option subnet-mask 255.255.255.240;
option routers 92.118.30.17;
option domain-name-servers 92.118.30.17;
option domain-name "house-ext.as205479.net";
}
}
subnet 192.168.10.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option routers 192.168.10.1;
option domain-name-servers 192.168.10.1;
option domain-name "eduroam.as205479.net";
default-lease-time 600;
max-lease-time 3600;
option interface-mtu 1420; # Wireguard
range 192.168.10.100 192.168.10.200;
}
'';
machines = [
{
hostName = "totoro";
ethernetAddress = "40:8d:5c:1f:e8:68";
ipAddress = "192.168.1.40";
}
{
hostName = "totoro-pfsense";
ethernetAddress = "52:54:00:cf:cd:94";
ipAddress = "192.168.1.41";
}
{
hostName = "kvm";
ethernetAddress = "00:0d:5d:1b:14:ba";
ipAddress = "192.168.1.50";
}
{
hostName = "printer-xerox";
ethernetAddress = "9c:93:4e:ad:1f:7b";
ipAddress = "192.168.1.51";
}
{
hostName = "ps5";
ethernetAddress = "bc:33:29:26:01:5c";
# This is used for DNAT on RTMP, above.
ipAddress = "92.118.30.18";
}
];
};
networking.wireguard = let
ifBase = {
listenPort = null;
allowedIPsAsRoutes = false;
};
peerBase = {
allowedIPs = [
"0.0.0.0/0"
"::/0"
];
};
in {
enable = true;
interfaces.wg-tuvok-vm = ifBase // {
ips = [
"2a09:a442::1:1/112"
"92.118.30.0/31"
];
listenPort = 51820;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "92.118.28.252:51820";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-vm fwmark 0xbeef
'';
};
interfaces.wg-tuvok-ee = ifBase // {
ips = [
"2a09:a442::2:1/112"
"92.118.30.2/31"
];
listenPort = 51821;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "[2a09:a441::f00f]:51821";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-ee fwmark 0xdead
'';
};
};
services.unifi = {
enable = true;
openFirewall = false;
unifiPackage = depot.pkgs.unifi;
};
services.prometheus.exporters.unifi-poller = {
enable = true;
controllers = [{
url = "https://localhost:8443";
verify_ssl = false;
user = "unifipoller";
pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; };
}];
};
networking.firewall = {
interfaces.en-general = {
allowedTCPPorts = [
8080 6789 # Unifi
53 # DNS
];
allowedUDPPorts = [
3478 10001 # Unifi
53 # DNS
];
};
interfaces.vl-eduroam = {
allowedTCPPorts = [
53 # DNS
];
allowedUDPPorts = [
53 # DNS
];
};
interfaces.wg-tuvok-ee = {
allowedUDPPorts = [
3784 # BFD
];
};
interfaces.wg-tuvok-vm = {
allowedUDPPorts = [
3784 # BFD
];
};
extraCommands = ''
ip46tables -F FORWARD
ip46tables -N ts-forward || true
ip46tables -A FORWARD -j ts-forward
iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-vm -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT
'';
};
services.ddclient = {
enable = false;
protocol = "cloudflare";
domains = ["home.lukegb.com"];
zone = "lukegb.com";
passwordFile = pkgs.writeText "cloudflare-token" secrets.cloudflareCredentials.token;
use = "if";
extraConfig = ''
if=en-virginmedia
daemon=0
'';
};
systemd.services.ddclient.serviceConfig.ExecStart = let
ddclient = pkgs.perlPackages.buildPerlPackage rec {
pname = "ddclient";
version = "3.9.1";
src = pkgs.fetchFromGitHub {
owner = "ddclient";
repo = "ddclient";
rev = "11a583b003920f8e15591813598b70061d1a4654";
sha256 = "sha256:1xz09vkii3mc2jmfwx9is07i06iiryv51571vdnl4m5mdnvsmlwb";
};
outputs = [ "out" ];
doCheck = false;
buildInputs = with pkgs.perlPackages; [ IOSocketSSL DigestSHA1 DataValidateIP JSONPP ];
nativeBuildInputs = with pkgs; [ autoreconfHook makeWrapper ];
preConfigure = ''
touch Makefile.PL
'';
postInstall = ''
patchShebangs $out/bin/ddclient
wrapProgram $out/bin/ddclient \
--suffix PATH : ${lib.makeBinPath (with pkgs; [ pkgs.iproute ])} \
--prefix PERL5LIB : $PERL5LIB
'';
};
RuntimeDirectory = "ddclient";
in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf";
environment.systemPackages = with pkgs; [
ethtool
];
services.coredns = {
enable = true;
config = ''
.:53 {
bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.253 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1
acl {
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24
block
}
hosts /dev/null {
#216.239.38.120 stadia.google.com stadia.com
fallthrough
}
loadbalance
forward . tls://8.8.8.8 tls://8.8.4.4 {
tls_servername dns.google
}
cache {
success 4096
denial 1024
prefetch 512
}
prometheus :9153
errors
log
}
'';
};
my.prometheus.additionalExporterPorts.coredns = 9153;
networking.resolvconf.extraConfig = ''
name_servers='127.0.0.253'
'';
services.prometheus.exporters.smokeping = {
enable = true;
hosts = [
"8.8.8.8" # Google Public DNS
"2001:4860:4860::8888"
"youtube.com" "ads.google.com" "google.com"
"1.1.1.1" # Cloudflare DNS
"2606:4700:4700::1111"
"twitter.com"
"store.steampowered.com"
"api.steampowered.com"
"euw1.api.riotgames.com" # League of Legends EUW
"eu.battle.net"
"185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone
"185.60.114.159" # Overwatch
# BFOB TS
"2a01:a500:85:3::2"
"37.9.61.53"
];
};
services.bird2 = {
enable = true;
config = ''
router id 92.118.30.254;
protocol kernel {
kernel table 150;
metric 0;
ipv4 {
import none;
export all;
};
};
protocol kernel {
kernel table 150;
metric 0;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 0.0.0.0/0 via 92.118.30.1 bfd {
# Virgin Media
preference = 100;
};
route 0.0.0.0/0 via 92.118.30.3 bfd {
# EE
preference = 10;
};
};
protocol static export6 {
ipv6 {};
route ::/0 via 2a09:a442::1:2 bfd {
# Virgin Media
preference = 100;
krt_prefsrc = 2a09:a443::1;
};
route ::/0 via 2a09:a442::2:2 bfd {
# EE
preference = 10;
krt_prefsrc = 2a09:a443::1;
};
# Covering route...
route 2a09:a443::/64 via "en-general";
route 2a09:a443:1::/48 via "en-general";
route 2a09:a443:2::/64 via "vl-eduroam";
route 2a09:a443:3::/48 via "vl-eduroam";
route 2a09:a443::/32 unreachable;
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.1;
neighbor 2a09:a442::1:2;
neighbor 92.118.30.3;
neighbor 2a09:a442::2:2;
};
'';
};
services.radvd = {
enable = true;
config = ''
interface en-general {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443::1 {};
DNSSL house.as205479.net {};
prefix 2a09:a443::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:1::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
interface vl-eduroam {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443:2::1 {};
DNSSL eduroam.as205479.net {};
prefix 2a09:a443:2::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:3::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
'';
};
services.dhcpd6 = {
enable = true;
interfaces = ["en-general" "vl-eduroam"];
authoritative = true;
extraConfig = ''
subnet6 2a09:a443:1::/48 {
range6 2a09:a443:1:1::/64;
range6 2a09:a443:1:2::/64 temporary;
prefix6 2a09:a443:1:1000:: 2a09:a443:1:ff00:: /56;
option dhcp6.name-servers 2a09:a443:1::1;
option dhcp6.domain-search "house.as205479.net";
}
subnet6 2a09:a443:3::/48 {
range6 2a09:a443:3:1::/64;
range6 2a09:a443:3:2::/64 temporary;
prefix6 2a09:a443:3:1000:: 2a09:a443:3:ff00:: /56;
option dhcp6.name-servers 2a09:a443:3::1;
option dhcp6.domain-search "eduroam.as205479.net";
}
'';
};
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
-web.listen-address 0.0.0.0:9324 \
-bird.socket /var/run/bird.ctl \
-bird.v2=true \
-format.new=true
'';
systemd.services.ee-scrape-data = let
scriptFile = ./ee-scrape-data.py;
python = pkgs.python3.withPackages (pm: with pm; [
requests
beautifulsoup4
html5lib
]);
in {
enable = true;
serviceConfig = {
Type = "oneshot";
ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
};
};
systemd.timers.ee-scrape-data = {
enable = true;
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitInactiveSec = "1m";
RandomizedDelaySec = "20";
};
};
system.stateVersion = "21.03";
}