# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0
{ depot, lib, pkgs, rebuilder, config, ... }:
let
inherit (depot.ops) secrets;
in {
imports = [
# We include this just so it sets some sysctls and firewall settings.
../lib/bgp.nix
];
boot.initrd.availableKernelModules = [
"sd_mod"
"ahci"
"usb_storage"
"usbhid"
];
boot.kernelParams = [ "mitigations=off" ];
fileSystems = {
"/" = {
device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
fsType = "ext4";
};
"/boot" = {
device = "/dev/disk/by-uuid/AB36-5BE4";
fsType = "vfat";
};
};
nix.maxJobs = lib.mkDefault 4;
# Use the systemd-boot EFI boot loader.
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
# Networking!
networking = {
# Routing tables:
# bgp (150) -- contains default routes over WG tunnels
# ee (201) -- table contains a static default route via EE
# main (254) -- DHCP routes (aka VM)
# Conventional lookup order is bgp, main. ee has routing rules.
hostName = "swann"; # Define your hostname.
domain = "int.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"];
useDHCP = false;
interfaces = {
lo = {
ipv4.addresses = [
{ address = "127.0.0.1"; prefixLength = 8; }
{ address = "92.118.30.254"; prefixLength = 32; }
];
};
en-virginmedia = {
useDHCP = true;
macAddress = "e4:3a:6e:16:07:61";
};
en-ee = {
ipv4.addresses = [
{ address = "192.168.200.2"; prefixLength = 24; }
];
ipv4.routes = [{
via = "192.168.200.1";
address = "0.0.0.0";
prefixLength = 0;
options.table = "201";
}];
};
en-general = {
ipv4.addresses = [
{ address = "192.168.1.1"; prefixLength = 23; }
];
ipv6.addresses = [
{ address = "2a09:a443::1"; prefixLength = 64; }
{ address = "2a09:a443:1::1"; prefixLength = 48; }
];
};
};
dhcpcd.extraConfig = ''
interface en-virginmedia
metric 100
interface en-ee
metric 250
'';
localCommands = let
claimedPriorities = { min = 10000; max = 10100; };
rules = [
# Route traffic to EE via WG... via EE.
{ priority = 10000; both = "fwmark 0xdead table 201"; }
# Route traffic to VM via WG... via VM DHCP in main table.
{ priority = 10001; both = "fwmark 0xbeef table main"; }
# Make ping work over the tunnels.
{ priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
{ priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
# Now some subset of RFC1918 via main table too.
{ priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
{ priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
{ priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
# And the linknets...
{ priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
{ priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
{ priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
# And the Google VIP I'm (ab)using for Stadia (see CoreDNS below).
{ priority = 10030; v4 = "to 216.239.38.120/32 table main"; }
# add-on.ee.co.uk goes via EE.
{ priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
# Anything originating from 192.168.200.0/24 should go via EE too.
{ priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
# Everything else over WG.
{ priority = 10099; both = "table 150"; }
];
clearRules = map (x: ''
ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
'') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
assert (both == "" || (v4 == "" && v6 == ""));
assert priority >= claimedPriorities.min;
assert priority < claimedPriorities.max;
let
rv4 = if v4 != "" then v4 else both;
rv6 = if v6 != "" then v6 else both;
in ''
${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
'';
addRules = map ruleToLine rules;
in ''
${lib.concatStringsSep "\n" clearRules}
${lib.concatStringsSep "\n" addRules}
ip -4 route flush table 151 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
ip -6 route flush table 151 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151
ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151
ip -4 route flush table 152 >/dev/null 2>&1 || true
ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151
ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
ip -6 route flush table 152 >/dev/null 2>&1 || true
ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152
'';
};
my.ip.tailscale = "100.102.224.95";
services.udev.extraRules = ''
ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
'';
boot.kernel.sysctl = {
"net.ipv4.ip_forward" = "1";
"net.ipv6.conf.default.forwarding" = "1";
"net.ipv6.conf.all.forwarding" = "1";
"net.ipv6.conf.en-virginmedia.accept_ra" = "2";
"net.ipv6.conf.en-ee.accept_ra" = "2";
};
networking.nat = {
enable = true;
internalInterfaces = ["en-general"];
externalInterface = "en-virginmedia";
extraCommands = ''
# NAT packets going over EE plain.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
# SNAT packets we're sending over tunnels.
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
'';
};
services.dhcpd4 = {
enable = true;
interfaces = ["en-general"];
authoritative = true;
extraConfig = ''
subnet 192.168.1.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option routers 192.168.1.1;
option domain-name-servers 192.168.1.1;
option domain-name "house.as205479.net";
default-lease-time 600;
max-lease-time 3600;
option interface-mtu 1420; # Wireguard
range 192.168.1.100 192.168.1.200;
}
'';
machines = [
{
hostName = "totoro";
ethernetAddress = "40:8d:5c:1f:e8:68";
ipAddress = "192.168.1.40";
}
{
hostName = "totoro-pfsense";
ethernetAddress = "52:54:00:cf:cd:94";
ipAddress = "192.168.1.41";
}
{
hostName = "kvm";
ethernetAddress = "00:0d:5d:1b:14:ba";
ipAddress = "192.168.1.50";
}
{
hostName = "printer-xerox";
ethernetAddress = "9c:93:4e:ad:1f:7b";
ipAddress = "192.168.1.51";
}
];
};
networking.wireguard = let
ifBase = {
listenPort = null;
allowedIPsAsRoutes = false;
};
peerBase = {
allowedIPs = [
"0.0.0.0/0"
"::/0"
];
};
in {
enable = true;
interfaces.wg-tuvok-vm = ifBase // {
ips = [
"2a09:a442::1:1/112"
"92.118.30.0/31"
];
listenPort = 51820;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "92.118.28.252:51820";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-vm fwmark 0xbeef
'';
};
interfaces.wg-tuvok-ee = ifBase // {
ips = [
"2a09:a442::2:1/112"
"92.118.30.2/31"
];
listenPort = 51821;
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
peers = [(peerBase // {
endpoint = "[2a09:a441::f00f]:51821";
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
})];
postSetup = ''
wg set wg-tuvok-ee fwmark 0xdead
'';
};
};
services.unifi = {
enable = true;
openPorts = false;
unifiPackage = depot.pkgs.unifi;
};
services.prometheus.exporters.unifi-poller = {
enable = true;
controllers = [{
url = "https://localhost:8443";
verify_ssl = false;
user = "unifipoller";
pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; };
}];
};
networking.firewall = {
interfaces.en-general = {
allowedTCPPorts = [
8080 6789 # Unifi
53 # DNS
];
allowedUDPPorts = [
3478 10001 # Unifi
53 # DNS
];
};
interfaces.wg-tuvok-ee = {
allowedUDPPorts = [
3784 # BFD
];
};
interfaces.wg-tuvok-vm = {
allowedUDPPorts = [
3784 # BFD
];
};
extraCommands = ''
iptables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
ip6tables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
'';
};
services.ddclient = {
enable = false;
protocol = "cloudflare";
domains = ["home.lukegb.com"];
zone = "lukegb.com";
password = secrets.cloudflareCredentials.token;
use = "if";
extraConfig = ''
if=en-virginmedia
daemon=0
'';
};
systemd.services.ddclient.serviceConfig.ExecStart = let
ddclient = pkgs.perlPackages.buildPerlPackage rec {
pname = "ddclient";
version = "3.9.1";
src = pkgs.fetchFromGitHub {
owner = "ddclient";
repo = "ddclient";
rev = "11a583b003920f8e15591813598b70061d1a4654";
sha256 = "sha256:1xz09vkii3mc2jmfwx9is07i06iiryv51571vdnl4m5mdnvsmlwb";
};
outputs = [ "out" ];
doCheck = false;
buildInputs = with pkgs.perlPackages; [ IOSocketSSL DigestSHA1 DataValidateIP JSONPP ];
nativeBuildInputs = with pkgs; [ autoreconfHook makeWrapper ];
preConfigure = ''
touch Makefile.PL
'';
postInstall = ''
patchShebangs $out/bin/ddclient
wrapProgram $out/bin/ddclient \
--suffix PATH : ${lib.makeBinPath (with pkgs; [ pkgs.iproute ])} \
--prefix PERL5LIB : $PERL5LIB
'';
};
RuntimeDirectory = "ddclient";
in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf";
environment.systemPackages = with pkgs; [
ethtool
];
services.coredns = {
enable = true;
config = ''
.:53 {
bind 192.168.1.1 127.0.0.53 2a09:a443::1 2a09:a443:1::1
acl {
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32
block
}
hosts /dev/null {
216.239.38.120 stadia.google.com stadia.com
fallthrough
}
loadbalance
forward . tls://8.8.8.8 tls://8.8.4.4 {
tls_servername dns.google
}
cache {
success 4096
denial 1024
prefetch 512
}
prometheus :9153
errors
log
}
'';
};
my.prometheus.additionalExporterPorts.coredns = 9153;
networking.resolvconf.extraConfig = ''
name_servers='127.0.0.53'
'';
services.prometheus.exporters.smokeping = {
enable = true;
hosts = [
"8.8.8.8" # Google Public DNS
"2001:4860:4860::8888"
"youtube.com" "ads.google.com" "google.com"
"1.1.1.1" # Cloudflare DNS
"2606:4700:4700::1111"
"twitter.com"
"store.steampowered.com"
"api.steampowered.com"
"prod.euw1.lol.riotgames.com" # League of Legends EUW
"eu.battle.net"
"185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone
"185.60.114.159" # Overwatch
# BFOB TS
"2a01:a500:85:3::2"
"37.9.61.53"
];
};
services.bird2 = {
enable = true;
config = ''
router id 92.118.30.254;
protocol kernel {
kernel table 150;
metric 0;
ipv4 {
import none;
export all;
};
};
protocol kernel {
kernel table 150;
metric 0;
ipv6 {
import none;
export all;
};
};
protocol device {};
protocol static export4 {
ipv4 {};
route 0.0.0.0/0 via 92.118.30.1 bfd {
# Virgin Media
preference = 100;
};
route 0.0.0.0/0 via 92.118.30.3 bfd {
# EE
preference = 10;
};
};
protocol static export6 {
ipv6 {};
route ::/0 via 2a09:a442::1:2 bfd {
# Virgin Media
preference = 100;
};
route ::/0 via 2a09:a442::2:2 bfd {
# EE
preference = 10;
};
# Covering route...
route 2a09:a443::/64 via "en-general";
route 2a09:a443::/32 unreachable;
};
protocol bfd {
interface "*" {
min rx interval 10ms;
min tx interval 50ms;
idle tx interval 1s;
multiplier 20;
};
neighbor 92.118.30.1;
neighbor 2a09:a442::1:2;
neighbor 92.118.30.3;
neighbor 2a09:a442::2:2;
};
'';
};
services.radvd = {
enable = true;
config = ''
interface en-general {
AdvSendAdvert on;
AdvLinkMTU 1420; # Wireguard
AdvManagedFlag on;
RDNSS 2a09:a443::1 {};
DNSSL house.as205479.net {};
prefix 2a09:a443::/64 {
AdvOnLink on;
AdvAutonomous on;
};
prefix 2a09:a443:1::/48 {
AdvOnLink on;
AdvAutonomous off;
};
};
'';
};
services.dhcpd6 = {
enable = true;
interfaces = ["en-general"];
authoritative = true;
extraConfig = ''
subnet6 2a09:a443:1::/48 {
range6 2a09:a443:1:1::/64;
range6 2a09:a443:1:2::/64 temporary;
prefix6 2a09:a443:1:1000:: 2a09:a443:1:ffff:: /56;
option dhcp6.name-servers 2a09:a443:1::1;
option dhcp6.domain-search "house.as205479.net";
}
'';
};
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
-web.listen-address 0.0.0.0:9324 \
-bird.socket /var/run/bird.ctl \
-bird.v2=true \
-format.new=true
'';
systemd.services.ee-scrape-data = let
scriptFile = ./ee-scrape-data.py;
python = pkgs.python3.withPackages (pm: with pm; [
requests
beautifulsoup4
html5lib
]);
in {
enable = true;
serviceConfig = {
Type = "oneshot";
ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
};
};
systemd.timers.ee-scrape-data = {
enable = true;
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnBootSec = "2m";
OnUnitInactiveSec = "1m";
RandomizedDelaySec = "20";
};
};
system.stateVersion = "21.03";
}