depot/ops/nixos/swann/default.nix

824 lines
24 KiB
Nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0
{ depot, lib, pkgs, config, ... }:
let
inherit (depot.ops) secrets;
inherit (lib) mkMerge mkForce;
in {
imports = [
# We include this just so it sets some sysctls and firewall settings.
../lib/bgp.nix
];
config = mkMerge [ {
boot.initrd.availableKernelModules = [
"sd_mod"
"ahci"
"usb_storage"
"usbhid"
];
boot.kernelParams = [ "mitigations=off" ];
fileSystems = {
"/" = {
device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
fsType = "ext4";
};
"/boot" = {
device = "/dev/disk/by-uuid/AB36-5BE4";
fsType = "vfat";
};
};
nix.settings.max-jobs = lib.mkDefault 4;
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
# Networking!
networking = {
# Routing tables:
# bgp (150) -- contains default routes over WG tunnels
# wg-ee (152) -- contains default routes over WG tunnels
# wg-gnet (153) -- contains default routes over WG tunnels
# ee (201) -- table contains a default route via EE
# vm (202) -- table contains a default route via VM
# gnet (203) -- table contains a default route via gnetwork
# main (254) -- basically empty
hostName = "swann"; # Define your hostname.
domain = "int.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"];
useNetworkd = true;
interfaces = {
lo = {
ipv4.addresses = [
{ address = "127.0.0.1"; prefixLength = 8; }
{ address = "92.118.30.254"; prefixLength = 32; }
{ address = "92.118.30.253"; prefixLength = 32; }
];
};
en-gnet = {
useDHCP = true;
ipv4.addresses = [
{ address = "192.168.201.2"; prefixLength = 24; }
];
# Additional options configured in networkd.
};
en-ee = {
useDHCP = true;
ipv4.addresses = [
{ address = "192.168.200.2"; prefixLength = 24; }
];
# Additional options configured in networkd.
};
br-internal = {
ipv4.addresses = [
{ address = "192.168.1.1"; prefixLength = 23; }
{ address = "92.118.30.17"; prefixLength = 28; }
];
ipv6.addresses = [
{ address = "2a09:a443::1"; prefixLength = 64; }
{ address = "2a09:a443:1::1"; prefixLength = 48; }
];
};
vl-eduroam = {
ipv4.addresses = [
{ address = "192.168.10.1"; prefixLength = 24; }
];
ipv6.addresses = [
{ address = "2a09:a443:2::1"; prefixLength = 64; }
{ address = "2a09:a443:3::1"; prefixLength = 48; }
];
};
};
};
systemd.network = let
hexToInt = h: (builtins.fromTOML "h = ${h}").h;
physicalNetwork = rtID: wireguardFwmark: extraRules: {
dhcpV4Config.RouteTable = rtID;
ipv6AcceptRAConfig.RouteTable = rtID;
routingPolicyRules = [{
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt wireguardFwmark;
Priority = 10000;
Table = rtID;
};
}] ++ extraRules;
};
wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: {
matchConfig.Name = linkName;
routes = let
replaceV4Octet = v4: fn: let
pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4;
in
"${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}";
replaceV6Octet = v6: fn: let
pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6;
in
"${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}";
in [
{
routeConfig = {
Destination = "${v4Linknet}/31";
Table = rtID;
};
}
{
routeConfig = {
Gateway = replaceV4Octet v4Linknet (n: n + 1);
Table = rtID;
};
}
{
routeConfig = {
Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112";
Table = rtID;
};
}
{
routeConfig = {
Gateway = replaceV6Octet v6Linknet (n: n + 1);
Table = rtID;
};
}
];
networkConfig = {
Address = [
"${v4Linknet}/31"
"${v6Linknet}/112"
];
};
routingPolicyRules = [
(tailscaleRule (relativePriority + 5000) rtID)
# Allow picking destination by source IP.
{
routingPolicyRuleConfig = {
Family = "ipv4";
From = v4Linknet;
Priority = 10010;
Table = rtID;
};
}
{
routingPolicyRuleConfig = {
Family = "ipv6";
From = v6Linknet;
Priority = 10010;
Table = rtID;
};
}
{
# Catch-all mop-up rule at the end.
routingPolicyRuleConfig = {
Family = "both";
Priority = relativePriority + 10090;
Table = rtID;
};
}
];
};
tailscaleRule = priority: table: {
# Route Tailscale (fwmark 0x80000) via Wireguard first.
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt "0x80000";
Priority = priority;
Table = table;
};
};
in let
routeTables = {
bgp = 150;
wg-ee = 152;
wg-gnet = 153;
ee = 201;
gnet = 203;
};
in {
enable = true;
config.routeTables = routeTables;
networks."50-wg-tuvok-ee" = wireguardNetwork {
linkName = "wg-tuvok-ee";
relativePriority = 3;
rtID = routeTables.wg-ee;
v4Linknet = "92.118.30.2";
v6Linknet = "2a09:a442::2:1";
};
networks."50-wg-tuvok-gnet" = wireguardNetwork {
linkName = "wg-tuvok-gnet";
relativePriority = 1;
rtID = routeTables.wg-gnet;
v4Linknet = "92.118.30.4";
v6Linknet = "2a09:a442::3:1";
};
networks."40-lo" = {
routingPolicyRules = let
viaMain = priority: to: {
routingPolicyRuleConfig = {
To = to;
Table = "main";
Priority = priority;
};
};
blackhole = fwmark: {
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt fwmark;
Priority = 10001;
Type = "unreachable";
};
};
in [
(tailscaleRule 5000 150)
# Blackhole connections that should be routed over individual interfaces.
(blackhole "0xdead")
(blackhole "0xbeef")
(blackhole "0xcafe")
# RFC 1918 via main table.
(viaMain 10020 "192.168.0.0/16")
(viaMain 10021 "10.0.0.0/8")
(viaMain 10022 "172.16.0.0/12")
# and the linknets.
(viaMain 10023 "92.118.30.0/24")
(viaMain 10024 "2a09:a442::1:0/112")
(viaMain 10025 "2a09:a442::2:0/112")
(viaMain 10026 "2a09:a442::3:0/112")
{
# Catch-all "go via WG"
routingPolicyRuleConfig = {
Family = "both";
Priority = 10080;
Table = routeTables.bgp;
};
}
];
};
networks."40-en-ee" = (physicalNetwork routeTables.ee "0xdead" [{
routingPolicyRuleConfig = {
# add-on.ee.co.uk goes via EE.
To = "82.192.97.153/32";
Table = routeTables.ee;
Priority = 10031;
};
} {
routingPolicyRuleConfig = {
# as does anything from 192.168.200.0/24.
From = "192.168.200.0/24";
Table = routeTables.ee;
Priority = 10031;
};
}]) // {
linkConfig.RequiredForOnline = "no";
};
networks."40-en-gnet" = (physicalNetwork routeTables.gnet "0xcafe" []);
networks."40-br-internal" = {
networkConfig.VLAN = [ "vl-eduroam" ];
};
networks."40-en-int-eth" = {
matchConfig.Name = "en-int-eth";
networkConfig.Bridge = "br-internal";
};
networks."40-en-int-sfp" = {
matchConfig.Name = "en-int-sfp";
networkConfig.Bridge = "br-internal";
};
netdevs = let
wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: {
netdevConfig = {
Name = name;
Kind = "wireguard";
Description = "WireGuard tunnel ${name}";
};
wireguardConfig = {
ListenPort = listenPort;
PrivateKeyFile = pkgs.writeText "${name}" privateKey;
# TODO: PrivateKeyFile
FirewallMark = hexToInt fwmark;
RouteTable = "off";
};
wireguardPeers = [{
wireguardPeerConfig = {
Endpoint = endpoint;
PublicKey = publicKey;
AllowedIPs = [
"0.0.0.0/0"
"::/0"
];
};
}];
};
tuvokWireguard = args: wireguard (args // {
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
});
in {
"40-wg-tuvok-ee" = tuvokWireguard {
name = "wg-tuvok-ee";
listenPort = 51821;
endpoint = "[2a09:a441::f00f]:51821";
fwmark = "0xdead";
};
"40-wg-tuvok-gnet" = tuvokWireguard {
name = "wg-tuvok-gnet";
listenPort = 51822;
endpoint = "92.118.28.252:51822";
fwmark = "0xcafe";
};
"20-br-internal" = {
netdevConfig = {
Name = "br-internal";
Kind = "bridge";
Description = "Bridge br-internal";
};
extraConfig = ''
[Bridge]
VLANFiltering=true
MulticastQuerier=true
MulticastSnooping=true
STP=true
VLANProtocol=802.1q
MulticastIGMPVersion=3
'';
};
"25-vl-eduroam" = {
netdevConfig = {
Name = "vl-eduroam";
Kind = "vlan";
Description = "Eduroam VLAN on br-internal";
};
vlanConfig = {
Id = 100;
};
};
};
};
services.mstpd.enable = true;
my.ip.tailscale = "100.102.224.95";
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6266:e05f";
services.udev.extraRules = ''
ATTR{address}=="e4:3a:6e:16:07:63", DRIVERS=="?*", NAME="en-ee"
ATTR{address}=="e4:3a:6e:16:07:64", DRIVERS=="?*", NAME="en-gnet"
ATTR{address}=="e4:3a:6e:16:07:67", DRIVERS=="?*", NAME="en-int-eth"
ATTR{address}=="e4:3a:6e:16:08:bc", DRIVERS=="?*", NAME="en-int-sfp"
'';
boot.kernel.sysctl = {
"net.ipv4.ip_forward" = "1";
"net.ipv6.conf.default.forwarding" = "1";
"net.ipv6.conf.all.forwarding" = "1";
"net.ipv6.conf.en-ee.accept_ra" = "2";
"net.ipv6.conf.en-gnet.accept_ra" = "2";
};
networking.nat = {
enable = true;
internalInterfaces = ["br-internal"];
externalInterface = "en-gnet";
extraCommands = ''
# Send PS5 RTMP to totoro instead.
# See DHCP static lease.
iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40
# NAT packets going over EE plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
# NAT packets going over GNetwork plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE
# SNAT packets we're sending over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254
# eduroam
# > mark incoming eduroam packets
iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2
# > NAT packets going out directly.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE
# > NAT packets going over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253
'';
};
services.dhcpd4 = {
enable = true;
interfaces = ["br-internal" "vl-eduroam"];
authoritative = true;
extraConfig = ''
shared-network int {
default-lease-time 3600;
max-lease-time 86400;
option interface-mtu 1420; # Wireguard
subnet 192.168.1.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option routers 192.168.1.1;
option domain-name-servers 192.168.1.1;
option domain-name "house.as205479.net";
range 192.168.1.100 192.168.1.200;
}
subnet 92.118.30.16 netmask 255.255.255.240 {
option subnet-mask 255.255.255.240;
option routers 92.118.30.17;
option domain-name-servers 92.118.30.17;
option domain-name "house-ext.as205479.net";
}
}
subnet 192.168.10.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option routers 192.168.10.1;
option domain-name-servers 192.168.10.1;
option domain-name "eduroam.as205479.net";
default-lease-time 600;
max-lease-time 3600;
option interface-mtu 1420; # Wireguard
range 192.168.10.100 192.168.10.200;
}
'';
machines = [
{
hostName = "totoro";
ethernetAddress = "40:8d:5c:1f:e8:68";
ipAddress = "192.168.1.40";
}
{
hostName = "totoro-pfsense";
ethernetAddress = "52:54:00:cf:cd:94";
ipAddress = "192.168.1.41";
}
{
hostName = "kvm";
ethernetAddress = "00:0d:5d:1b:14:ba";
ipAddress = "192.168.1.50";
}
{
hostName = "printer-xerox";
ethernetAddress = "9c:93:4e:ad:1f:7b";
ipAddress = "192.168.1.51";
}
{
hostName = "ps5";
ethernetAddress = "bc:33:29:26:01:5c";
# This is used for DNAT on RTMP, above.
ipAddress = "92.118.30.18";
}
];
};
systemd.services.dhcpd4 = {
wants = [ "systemd-networkd-wait-online.service" ];
after = [ "systemd-networkd-wait-online.service" ];
};
networking.firewall = {
interfaces.br-internal = {
allowedTCPPorts = [
8080 6789 # Unifi
53 # DNS
];
allowedUDPPorts = [
3478 10001 # Unifi
53 # DNS
];
};
interfaces.vl-eduroam = {
allowedTCPPorts = [
53 # DNS
];
allowedUDPPorts = [
53 # DNS
];
};
interfaces.en-ee = {
allowedUDPPorts = [
51821
];
};
interfaces.en-gnet = {
allowedUDPPorts = [
51822
];
};
interfaces.wg-tuvok-ee = {
allowedUDPPorts = [
3784 # BFD
];
};
interfaces.wg-tuvok-gnet = {
allowedUDPPorts = [
3784 # BFD
];
};
extraCommands = ''
ip46tables -F FORWARD
ip46tables -N ts-forward || true
ip46tables -A FORWARD -j ts-forward
iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT
'';
};
environment.systemPackages = with pkgs; [
ethtool
(writeShellApplication {
name = "bridge-stp";
runtimeInputs = [ mstpd ];
text = ''
BRIDGES=("br-internal")
for BRIDGE in "''${BRIDGES[@]}"; do
if [[ "$BRIDGE" = "$1" ]]; then
if [[ "$2" = "start" ]]; then
mstpctl addbridge "$BRIDGE"
exit 0
elif [[ "$2" = "stop" ]]; then
mstpctl delbridge "$BRIDGE"
exit 0
fi
exit 1
fi
done
exit 1
'';
})
];
services.coredns = {
enable = true;
config = ''
.:53 {
bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.253 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1
acl {
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24
block
}
hosts /dev/null {
#216.239.38.120 stadia.google.com stadia.com
fallthrough
}
loadbalance
forward . tls://8.8.8.8 tls://8.8.4.4 {
tls_servername dns.google
}
cache {
success 4096
denial 1024
prefetch 512
}
prometheus :9153
errors
log
}
'';
};
systemd.services.coredns = {
wants = [ "systemd-networkd-wait-online.service" ];
after = [ "systemd-networkd-wait-online.service" ];
};
my.prometheus.additionalExporterPorts.coredns = 9153;
networking.resolvconf.extraConfig = ''
name_servers='127.0.0.253'
'';
services.prometheus.exporters.smokeping = {
enable = true;
hosts = [
"8.8.8.8" # Google Public DNS
"2001:4860:4860::8888"
"youtube.com" "ads.google.com" "google.com"
"1.1.1.1" # Cloudflare DNS
"2606:4700:4700::1111"
"twitter.com"
"store.steampowered.com"
"api.steampowered.com"
# "euw1.api.riotgames.com" # League of Legends EUW
"eu.battle.net"
"185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone
"185.60.114.159" # Overwatch
# BFOB TS
"2a01:a500:85:3::2"
"37.9.61.53"
# lukegb01.ring.nlnog.net
"2a09:a441::13"
"92.118.28.13"
];
};
services.bird2 = {
enable = true;
config = ''
router id 92.118.30.254;
protocol kernel {
kernel table 150;
metric 0;
ipv4 {
import none;
export all;
};
};
protocol kernel {
kernel table 150;
metric 0;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 0.0.0.0/0 via 92.118.30.1 bfd {
# Virgin Media
preference = 100;
};
route 0.0.0.0/0 via 92.118.30.3 bfd {
# EE
preference = 10;
};
route 0.0.0.0/0 via 92.118.30.5 bfd {
# GNetwork
preference = 200;
};
};
protocol static export6 {
ipv6 {};
route ::/0 via 2a09:a442::1:2 bfd {
# Virgin Media
preference = 100;
krt_prefsrc = 2a09:a443::1;
};
route ::/0 via 2a09:a442::2:2 bfd {
# EE
preference = 10;
krt_prefsrc = 2a09:a443::1;
};
route ::/0 via 2a09:a442::3:2 bfd {
# GNetwork
preference = 200;
krt_prefsrc = 2a09:a443::1;
};
# Covering route...
route 2a09:a443::/64 via "br-internal";
route 2a09:a443:1::/48 via "br-internal";
route 2a09:a443:2::/64 via "vl-eduroam";
route 2a09:a443:3::/48 via "vl-eduroam";
route 2a09:a443::/32 unreachable;
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.1;
neighbor 2a09:a442::1:2;
neighbor 92.118.30.3;
neighbor 2a09:a442::2:2;
neighbor 92.118.30.5;
neighbor 2a09:a442::3:2;
};
'';
};
services.radvd = {
enable = true;
config = ''
interface br-internal {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443::1 {};
DNSSL house.as205479.net {};
prefix 2a09:a443::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:1::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
interface vl-eduroam {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443:2::1 {};
DNSSL eduroam.as205479.net {};
prefix 2a09:a443:2::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:3::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
'';
};
services.dhcpd6 = {
enable = true;
interfaces = ["br-internal" "vl-eduroam"];
authoritative = true;
extraConfig = ''
subnet6 2a09:a443:1::/48 {
range6 2a09:a443:1:1::/64;
range6 2a09:a443:1:2::/64 temporary;
prefix6 2a09:a443:1:1000:: 2a09:a443:1:ff00:: /56;
option dhcp6.name-servers 2a09:a443:1::1;
option dhcp6.domain-search "house.as205479.net";
}
subnet6 2a09:a443:3::/48 {
range6 2a09:a443:3:1::/64;
range6 2a09:a443:3:2::/64 temporary;
prefix6 2a09:a443:3:1000:: 2a09:a443:3:ff00:: /56;
option dhcp6.name-servers 2a09:a443:3::1;
option dhcp6.domain-search "eduroam.as205479.net";
}
'';
};
systemd.services.dhcpd6 = {
wants = [ "systemd-networkd-wait-online.service" ];
after = [ "systemd-networkd-wait-online.service" ];
};
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
-web.listen-address 0.0.0.0:9324 \
-bird.socket /var/run/bird.ctl \
-bird.v2=true \
-format.new=true
'';
systemd.services.ee-scrape-data = let
scriptFile = ./ee-scrape-data.py;
python = pkgs.python3.withPackages (pm: with pm; [
requests
beautifulsoup4
html5lib
]);
in {
enable = true;
serviceConfig = {
Type = "oneshot";
ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
};
};
systemd.timers.ee-scrape-data = {
enable = true;
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitInactiveSec = "1m";
RandomizedDelaySec = "20";
};
};
system.stateVersion = "21.03";
} {
# Minimize writes to storage.
boot.tmpOnTmpfs = true;
services.journald.extraConfig = ''
Storage=volatile
'';
systemd.services.tailscaled.environment.TS_LOGS_DIR = "/var/run/tailscale";
} ];
}