# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0
{ depot, lib, pkgs, config, ... }:
let
inherit (depot.ops) secrets;
inherit (lib) mkMerge mkForce;
in {
imports = [
# We include this just so it sets some sysctls and firewall settings.
../lib/bgp.nix
../lib/erbium.nix
];
config = mkMerge [ {
# TODO: erbium is broken; don't deploy!!!
my.deploy.enable = false;
boot.initrd.availableKernelModules = [
"sd_mod"
"ahci"
"usb_storage"
"usbhid"
];
boot.kernelParams = [ "mitigations=off" ];
fileSystems = {
"/" = {
device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
fsType = "ext4";
};
"/boot" = {
device = "/dev/disk/by-uuid/AB36-5BE4";
fsType = "vfat";
};
};
nix.settings.max-jobs = lib.mkDefault 4;
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
# Networking!
networking = {
# Routing tables:
# bgp (150) -- contains default routes over WG tunnels
# wg-ee (152) -- contains default routes over WG tunnels
# wg-gnet (153) -- contains default routes over WG tunnels
# ee (201) -- table contains a default route via EE
# vm (202) -- table contains a default route via VM
# gnet (203) -- table contains a default route via gnetwork
# main (254) -- basically empty
hostName = "swann"; # Define your hostname.
domain = "int.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"];
useNetworkd = true;
interfaces = {
lo = {
ipv4.addresses = [
{ address = "127.0.0.1"; prefixLength = 8; }
{ address = "92.118.30.254"; prefixLength = 32; }
{ address = "92.118.30.253"; prefixLength = 32; }
];
};
en-gnet = {
useDHCP = true;
ipv4.addresses = [
{ address = "192.168.201.2"; prefixLength = 24; }
];
# Additional options configured in networkd.
};
en-ee = {
useDHCP = true;
ipv4.addresses = [
{ address = "192.168.200.2"; prefixLength = 24; }
];
# Additional options configured in networkd.
};
br-internal = {
ipv4.addresses = [
{ address = "192.168.1.1"; prefixLength = 23; }
{ address = "92.118.30.17"; prefixLength = 28; }
];
ipv6.addresses = [
{ address = "2a09:a443::1"; prefixLength = 64; }
{ address = "2a09:a443:1::1"; prefixLength = 48; }
];
};
vl-eduroam = {
ipv4.addresses = [
{ address = "192.168.10.1"; prefixLength = 24; }
];
ipv6.addresses = [
{ address = "2a09:a443:2::1"; prefixLength = 64; }
{ address = "2a09:a443:3::1"; prefixLength = 48; }
];
};
};
};
systemd.network = let
hexToInt = h: (builtins.fromTOML "h = ${h}").h;
physicalNetwork = rtID: wireguardFwmark: extraRules: {
dhcpV4Config.RouteTable = rtID;
ipv6AcceptRAConfig.RouteTable = rtID;
routingPolicyRules = [{
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt wireguardFwmark;
Priority = 10000;
Table = rtID;
};
}] ++ extraRules;
};
wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: {
matchConfig.Name = linkName;
routes = let
replaceV4Octet = v4: fn: let
pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4;
in
"${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}";
replaceV6Octet = v6: fn: let
pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6;
in
"${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}";
in [
{
routeConfig = {
Destination = "${v4Linknet}/31";
Table = rtID;
};
}
{
routeConfig = {
Gateway = replaceV4Octet v4Linknet (n: n + 1);
Table = rtID;
};
}
{
routeConfig = {
Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112";
Table = rtID;
};
}
{
routeConfig = {
Gateway = replaceV6Octet v6Linknet (n: n + 1);
Table = rtID;
};
}
];
networkConfig = {
Address = [
"${v4Linknet}/31"
"${v6Linknet}/112"
];
};
routingPolicyRules = [
(tailscaleRule (relativePriority + 5000) rtID)
# Allow picking destination by source IP.
{
routingPolicyRuleConfig = {
Family = "ipv4";
From = v4Linknet;
Priority = 10010;
Table = rtID;
};
}
{
routingPolicyRuleConfig = {
Family = "ipv6";
From = v6Linknet;
Priority = 10010;
Table = rtID;
};
}
];
};
tailscaleRule = priority: table: {
# Route Tailscale (fwmark 0x80000) via Wireguard first.
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt "0x80000"; # Should be "0x80000/0xff0000"
Priority = priority;
Table = table;
};
};
in let
routeTables = {
bgp = 150;
wg-ee = 152;
wg-gnet = 153;
ee = 201;
gnet = 203;
};
in {
enable = true;
config.routeTables = routeTables;
networks."50-wg-tuvok-ee" = wireguardNetwork {
linkName = "wg-tuvok-ee";
relativePriority = 3;
rtID = routeTables.wg-ee;
v4Linknet = "92.118.30.2";
v6Linknet = "2a09:a442::2:1";
};
networks."50-wg-tuvok-gnet" = wireguardNetwork {
linkName = "wg-tuvok-gnet";
relativePriority = 1;
rtID = routeTables.wg-gnet;
v4Linknet = "92.118.30.4";
v6Linknet = "2a09:a442::3:1";
};
networks."40-lo" = {
routingPolicyRules = let
viaMain = priority: to: {
routingPolicyRuleConfig = {
To = to;
Table = "main";
Priority = priority;
};
};
blackhole = fwmark: {
routingPolicyRuleConfig = {
Family = "both";
FirewallMark = hexToInt fwmark;
Priority = 10001;
Type = "unreachable";
};
};
in [
(tailscaleRule 5000 150)
# Blackhole connections that should be routed over individual interfaces.
(blackhole "0xdead")
(blackhole "0xbeef")
(blackhole "0xcafe")
# RFC 1918 via main table.
(viaMain 10020 "192.168.0.0/16")
(viaMain 10021 "10.0.0.0/8")
(viaMain 10022 "172.16.0.0/12")
# and the linknets.
(viaMain 10023 "92.118.30.0/24")
(viaMain 10024 "2a09:a442::1:0/112")
(viaMain 10025 "2a09:a442::2:0/112")
(viaMain 10026 "2a09:a442::3:0/112")
{
# Catch-all "go via WG"
routingPolicyRuleConfig = {
Family = "both";
Priority = 10080;
Table = routeTables.bgp;
};
}
];
};
networks."40-en-ee" = (physicalNetwork routeTables.ee "0xdead" [{
routingPolicyRuleConfig = {
# add-on.ee.co.uk goes via EE.
To = "82.192.97.153/32";
Table = routeTables.ee;
Priority = 10031;
};
} {
routingPolicyRuleConfig = {
# as does anything from 192.168.200.0/24.
From = "192.168.200.0/24";
Table = routeTables.ee;
Priority = 10031;
};
}]) // {
linkConfig.RequiredForOnline = "no";
};
networks."40-en-gnet" = (physicalNetwork routeTables.gnet "0xcafe" [{
# Catch-all mop-up rule at the end.
routingPolicyRuleConfig = {
Family = "both";
Priority = 32768;
Table = routeTables.gnet;
};
}]);
networks."40-br-internal" = {
networkConfig.VLAN = [ "vl-eduroam" ];
};
networks."40-en-int-eth" = {
matchConfig.Name = "en-int-eth";
networkConfig.Bridge = "br-internal";
};
networks."40-en-int-sfp" = {
matchConfig.Name = "en-int-sfp";
networkConfig.Bridge = "br-internal";
};
netdevs = let
wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: {
netdevConfig = {
Name = name;
Kind = "wireguard";
Description = "WireGuard tunnel ${name}";
};
wireguardConfig = {
ListenPort = listenPort;
PrivateKeyFile = pkgs.writeText "${name}" privateKey;
# TODO: PrivateKeyFile
FirewallMark = hexToInt fwmark;
RouteTable = "off";
};
wireguardPeers = [{
wireguardPeerConfig = {
Endpoint = endpoint;
PublicKey = publicKey;
AllowedIPs = [
"0.0.0.0/0"
"::/0"
];
};
}];
};
tuvokWireguard = args: wireguard (args // {
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
});
in {
"40-wg-tuvok-ee" = tuvokWireguard {
name = "wg-tuvok-ee";
listenPort = 51821;
endpoint = "[2a09:a441::f00f]:51821";
fwmark = "0xdead";
};
"40-wg-tuvok-gnet" = tuvokWireguard {
name = "wg-tuvok-gnet";
listenPort = 51822;
endpoint = "92.118.28.252:51822";
fwmark = "0xcafe";
};
"20-br-internal" = {
netdevConfig = {
Name = "br-internal";
Kind = "bridge";
Description = "Bridge br-internal";
};
extraConfig = ''
[Bridge]
VLANFiltering=true
MulticastQuerier=true
MulticastSnooping=true
STP=true
VLANProtocol=802.1q
MulticastIGMPVersion=3
'';
};
"25-vl-eduroam" = {
netdevConfig = {
Name = "vl-eduroam";
Kind = "vlan";
Description = "Eduroam VLAN on br-internal";
};
vlanConfig = {
Id = 100;
};
};
};
};
services.mstpd.enable = true;
my.ip.tailscale = "100.102.224.95";
my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6266:e05f";
services.udev.extraRules = ''
ATTR{address}=="e4:3a:6e:16:07:63", DRIVERS=="?*", NAME="en-ee"
ATTR{address}=="e4:3a:6e:16:07:64", DRIVERS=="?*", NAME="en-gnet"
ATTR{address}=="e4:3a:6e:16:07:67", DRIVERS=="?*", NAME="en-int-eth"
ATTR{address}=="e4:3a:6e:16:08:bc", DRIVERS=="?*", NAME="en-int-sfp"
'';
boot.kernel.sysctl = {
"net.ipv4.ip_forward" = "1";
"net.ipv6.conf.default.forwarding" = "1";
"net.ipv6.conf.all.forwarding" = "1";
"net.ipv6.conf.en-ee.accept_ra" = "2";
"net.ipv6.conf.en-gnet.accept_ra" = "2";
};
networking.nat = {
enable = true;
internalInterfaces = ["br-internal"];
externalInterface = "en-gnet";
extraCommands = ''
# Send PS5 RTMP to totoro instead.
# See DHCP static lease.
iptables -w -t nat -A nixos-nat-pre --src 92.118.30.18 -p tcp --dport 1935 -j DNAT --to-destination 192.168.1.40
# NAT packets going over EE plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
# NAT packets going over GNetwork plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE
# SNAT packets we're sending over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254
# eduroam
# > mark incoming eduroam packets
iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2
# > NAT packets going out directly.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE
# > NAT packets going over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253
iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253
'';
};
services.erbium = {
enable = true;
package = depot.nix.pkgs.erbium;
settings = {
addresses = [
# internal
"192.168.1.0/24" "92.118.30.16/28" "2a09:a443::/64" "2a09:a443:1::/48"
# eduroam
"192.168.10.0/24" "2a09:a443:2::/64" "2a09:a443:3::/48"
];
dns-servers = [ "$self4" "$self6" ];
api-listeners = [ "[::1]:9968" ];
dns-listeners = [ "[::1]:11153" ]; # if we don't specify something then erbium crashes
router-advertisements = let
baseline = {
mtu = 1420;
lifetime = "1h";
reachable = "20m";
};
baselinePrefix = {
on-link = true;
autonomous = true;
valid = "30d";
preferred = "7d";
};
in {
br-internal = baseline // {
dns-servers.addresses = [ "2a09:a443::1" ];
dns-search.domains = [ "house.as205479.net" ];
prefixes = [(baselinePrefix // {
prefix = "2a09:a443::/64";
}) (baselinePrefix // {
prefix = "2a09:a443:1::/48";
autonomous = false;
})];
};
vl-eduroam = baseline // {
dns-servers.addresses = [ "2a09:a443:2::1" ];
dns-search.domains = [ "eduroam.as205479.net" ];
prefixes = [(baselinePrefix // {
prefix = "2a09:a443:2::/64";
}) (baselinePrefix // {
prefix = "2a09:a443:3::/48";
autonomous = false;
})];
};
};
dhcp-policies = [
# public internal
{
apply-subnet = "92.118.30.16/28";
apply-domain-name = "house-ext.as205479.net";
apply-dns-servers = [ "92.118.30.17" ];
apply-routers = [ "92.118.30.17" ];
apply-mtu = 1420;
policies = [{
match-hardware-address = "bc:33:29:26:01:5c";
apply-host-name = "ps5";
apply-address = "92.118.30.18";
}];
}
# private internal
{
match-subnet = "192.168.1.0/24";
apply-range.start = "192.168.1.100";
apply-range.end = "192.168.1.200";
apply-domain-name = "house.as205479.net";
apply-dns-servers = [ "192.168.1.1" ];
apply-routers = [ "192.168.1.1" ];
apply-mtu = 1420;
policies = [{
match-hardware-address = "40:8d:5c:1f:e8:68";
apply-host-name = "totoro";
apply-address = "192.168.1.40";
} {
match-hardware-address = "52:54:00:cf:cd:94";
apply-host-name = "totoro-pfsense";
apply-address = "192.168.1.41";
} {
match-hardware-address = "00:0d:5d:1b:14:ba";
apply-host-name = "kvm";
apply-address = "192.168.1.50";
} {
match-hardware-address = "9c:93:4e:ad:1f:7b";
apply-host-name = "printer-xerox";
apply-address = "192.168.1.51";
} {
match-hardware-address = "84:39:be:77:65:52";
apply-host-name = "qvmpc6552";
apply-address = "192.168.1.60";
}];
}
# eduroam
{
match-subnet = "192.168.10.0/24";
apply-range.start = "192.168.10.10";
apply-range.end = "192.168.10.200";
apply-domain-name = "eduroam.as205479.net";
apply-dns-servers = [ "192.168.10.1" ];
apply-routers = [ "192.168.10.1" ];
apply-mtu = 1420;
}
];
};
};
networking.firewall = {
interfaces.br-internal = {
allowedTCPPorts = [
8080 6789 # Unifi
53 # DNS
];
allowedUDPPorts = [
3478 10001 # Unifi
53 # DNS
];
};
interfaces.vl-eduroam = {
allowedTCPPorts = [
53 # DNS
];
allowedUDPPorts = [
53 # DNS
];
};
interfaces.en-ee = {
allowedUDPPorts = [
51821
];
};
interfaces.en-gnet = {
allowedUDPPorts = [
51822
];
};
interfaces.wg-tuvok-ee = {
allowedUDPPorts = [
3784 # BFD
];
};
interfaces.wg-tuvok-gnet = {
allowedUDPPorts = [
3784 # BFD
];
};
extraCommands = ''
ip46tables -F FORWARD
ip46tables -N ts-forward || true
ip46tables -A FORWARD -j ts-forward
iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT
ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT
'';
};
environment.systemPackages = with pkgs; [
ethtool
(writeShellApplication {
name = "bridge-stp";
runtimeInputs = [ mstpd ];
text = ''
BRIDGES=("br-internal")
for BRIDGE in "''${BRIDGES[@]}"; do
if [[ "$BRIDGE" = "$1" ]]; then
if [[ "$2" = "start" ]]; then
mstpctl addbridge "$BRIDGE"
exit 0
elif [[ "$2" = "stop" ]]; then
mstpctl delbridge "$BRIDGE"
exit 0
fi
exit 1
fi
done
exit 1
'';
})
];
services.coredns = {
enable = true;
config = ''
.:53 {
bind 192.168.1.1 92.118.30.17 192.168.10.1 127.0.0.253 2a09:a443::1 2a09:a443:1::1 2a09:a443:2::1 2a09:a443:3::1
acl {
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 92.118.30.0/24
block
}
hosts /dev/null {
#216.239.38.120 stadia.google.com stadia.com
fallthrough
}
loadbalance
forward . tls://8.8.8.8 tls://8.8.4.4 {
tls_servername dns.google
}
cache {
success 4096
denial 1024
prefetch 512
}
prometheus :9153
errors
log
}
'';
};
systemd.services.coredns = {
wants = [ "systemd-networkd-wait-online.service" ];
after = [ "systemd-networkd-wait-online.service" ];
};
my.prometheus.additionalExporterPorts.coredns = 9153;
networking.resolvconf.extraConfig = ''
name_servers='127.0.0.253'
'';
services.prometheus.exporters.smokeping = {
enable = true;
hosts = [
"8.8.8.8" # Google Public DNS
"2001:4860:4860::8888"
"youtube.com" "ads.google.com" "google.com"
"1.1.1.1" # Cloudflare DNS
"2606:4700:4700::1111"
"twitter.com"
"store.steampowered.com"
"api.steampowered.com"
# "euw1.api.riotgames.com" # League of Legends EUW
"eu.battle.net"
"185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone
"185.60.114.159" # Overwatch
# BFOB TS
"2a01:a500:85:3::2"
"37.9.61.53"
# lukegb01.ring.nlnog.net
"2a09:a441::13"
"92.118.28.13"
];
};
services.bird2 = {
enable = true;
config = ''
router id 92.118.30.254;
protocol kernel {
kernel table 150;
metric 0;
ipv4 {
import none;
export all;
};
};
protocol kernel {
kernel table 150;
metric 0;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 0.0.0.0/0 via 92.118.30.1 bfd {
# Virgin Media
preference = 100;
};
route 0.0.0.0/0 via 92.118.30.3 bfd {
# EE
preference = 10;
};
route 0.0.0.0/0 via 92.118.30.5 bfd {
# GNetwork
preference = 200;
};
};
protocol static export6 {
ipv6 {};
route ::/0 via 2a09:a442::1:2 bfd {
# Virgin Media
preference = 100;
krt_prefsrc = 2a09:a443::1;
};
route ::/0 via 2a09:a442::2:2 bfd {
# EE
preference = 10;
krt_prefsrc = 2a09:a443::1;
};
route ::/0 via 2a09:a442::3:2 bfd {
# GNetwork
preference = 200;
krt_prefsrc = 2a09:a443::1;
};
# Covering route...
route 2a09:a443::/64 via "br-internal";
route 2a09:a443:1::/48 via "br-internal";
route 2a09:a443:2::/64 via "vl-eduroam";
route 2a09:a443:3::/48 via "vl-eduroam";
route 2a09:a443::/32 unreachable;
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.1;
neighbor 2a09:a442::1:2;
neighbor 92.118.30.3;
neighbor 2a09:a442::2:2;
neighbor 92.118.30.5;
neighbor 2a09:a442::3:2;
};
'';
};
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
-web.listen-address 0.0.0.0:9324 \
-bird.socket /var/run/bird.ctl \
-bird.v2=true \
-format.new=true
'';
systemd.services.ee-scrape-data = let
scriptFile = ./ee-scrape-data.py;
python = pkgs.python3.withPackages (pm: with pm; [
requests
beautifulsoup4
html5lib
]);
in {
enable = true;
serviceConfig = {
Type = "oneshot";
ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
};
};
systemd.timers.ee-scrape-data = {
enable = true;
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitInactiveSec = "1m";
RandomizedDelaySec = "20";
};
};
system.stateVersion = "21.03";
} {
# Minimize writes to storage.
boot.tmp.useTmpfs = true;
services.journald.extraConfig = ''
Storage=volatile
'';
systemd.services.tailscaled.environment.TS_LOGS_DIR = "/var/run/tailscale";
} ];
}