Luke Granger-Brown
4bb015ee0d
EE uses CGNAT on IPv4, which makes this... less than ideal. However, IPv6 is IPv6 and works pretty reasonably.
552 lines
15 KiB
Nix
552 lines
15 KiB
Nix
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
{ depot, lib, pkgs, rebuilder, config, ... }:
|
|
let
|
|
inherit (depot.ops) secrets;
|
|
in {
|
|
imports = [
|
|
# We include this just so it sets some sysctls and firewall settings.
|
|
../lib/bgp.nix
|
|
];
|
|
|
|
boot.initrd.availableKernelModules = [
|
|
"sd_mod"
|
|
"ahci"
|
|
"usb_storage"
|
|
"usbhid"
|
|
];
|
|
boot.kernelParams = [ "mitigations=off" ];
|
|
|
|
fileSystems = {
|
|
"/" = {
|
|
device = "/dev/disk/by-uuid/fc964ef6-e3d0-4472-bc0e-f96f977ebf11";
|
|
fsType = "ext4";
|
|
};
|
|
"/boot" = {
|
|
device = "/dev/disk/by-uuid/AB36-5BE4";
|
|
fsType = "vfat";
|
|
};
|
|
};
|
|
|
|
nix.maxJobs = lib.mkDefault 4;
|
|
|
|
# Use the systemd-boot EFI boot loader.
|
|
boot.loader.systemd-boot.enable = true;
|
|
boot.loader.efi.canTouchEfiVariables = true;
|
|
|
|
# Networking!
|
|
networking = {
|
|
# Routing tables:
|
|
# bgp (150) -- contains default routes over WG tunnels
|
|
# ee (201) -- table contains a static default route via EE
|
|
# main (254) -- DHCP routes (aka VM)
|
|
# Conventional lookup order is bgp, main. ee has routing rules.
|
|
|
|
hostName = "swann"; # Define your hostname.
|
|
domain = "int.as205479.net";
|
|
nameservers = ["8.8.8.8" "8.8.4.4"];
|
|
useDHCP = false;
|
|
interfaces = {
|
|
lo = {
|
|
ipv4.addresses = [
|
|
{ address = "127.0.0.1"; prefixLength = 8; }
|
|
{ address = "92.118.30.254"; prefixLength = 32; }
|
|
];
|
|
};
|
|
en-virginmedia = {
|
|
useDHCP = true;
|
|
macAddress = "e4:3a:6e:16:07:61";
|
|
};
|
|
en-ee = {
|
|
ipv4.addresses = [
|
|
{ address = "192.168.200.2"; prefixLength = 24; }
|
|
];
|
|
ipv4.routes = [{
|
|
via = "192.168.200.1";
|
|
address = "0.0.0.0";
|
|
prefixLength = 0;
|
|
options.table = "201";
|
|
}];
|
|
};
|
|
en-general = {
|
|
ipv4.addresses = [
|
|
{ address = "192.168.1.1"; prefixLength = 23; }
|
|
];
|
|
ipv6.addresses = [
|
|
{ address = "2a09:a443::1"; prefixLength = 64; }
|
|
{ address = "2a09:a443:1::1"; prefixLength = 48; }
|
|
];
|
|
};
|
|
};
|
|
|
|
dhcpcd.extraConfig = ''
|
|
interface en-virginmedia
|
|
metric 100
|
|
|
|
interface en-ee
|
|
metric 250
|
|
'';
|
|
localCommands = let
|
|
claimedPriorities = { min = 10000; max = 10100; };
|
|
rules = [
|
|
# Route traffic to EE via WG... via EE.
|
|
{ priority = 10000; both = "fwmark 0xdead table 201"; }
|
|
# Route traffic to VM via WG... via VM DHCP in main table.
|
|
{ priority = 10001; both = "fwmark 0xbeef table main"; }
|
|
|
|
# Make ping work over the tunnels.
|
|
{ priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
|
|
{ priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
|
|
|
|
# Now some subset of RFC1918 via main table too.
|
|
{ priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
|
|
{ priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
|
|
{ priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
|
|
|
|
# And the linknets...
|
|
{ priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
|
|
{ priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
|
|
{ priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
|
|
|
|
# And the Google VIP I'm (ab)using for Stadia (see CoreDNS below).
|
|
{ priority = 10030; v4 = "to 216.239.38.120/32 table main"; }
|
|
|
|
# add-on.ee.co.uk goes via EE.
|
|
{ priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
|
|
|
|
# Anything originating from 192.168.200.0/24 should go via EE too.
|
|
{ priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
|
|
|
|
# Everything else over WG.
|
|
{ priority = 10099; both = "table 150"; }
|
|
];
|
|
clearRules = map (x: ''
|
|
ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
|
|
ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
|
|
'') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
|
|
ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
|
|
assert (both == "" || (v4 == "" && v6 == ""));
|
|
assert priority >= claimedPriorities.min;
|
|
assert priority < claimedPriorities.max;
|
|
let
|
|
rv4 = if v4 != "" then v4 else both;
|
|
rv6 = if v6 != "" then v6 else both;
|
|
in ''
|
|
${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
|
|
${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
|
|
'';
|
|
addRules = map ruleToLine rules;
|
|
in ''
|
|
${lib.concatStringsSep "\n" clearRules}
|
|
${lib.concatStringsSep "\n" addRules}
|
|
|
|
ip -4 route flush table 151 >/dev/null 2>&1 || true
|
|
ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
|
|
ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
|
|
ip -6 route flush table 151 >/dev/null 2>&1 || true
|
|
ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151
|
|
ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151
|
|
|
|
ip -4 route flush table 152 >/dev/null 2>&1 || true
|
|
ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151
|
|
ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
|
|
ip -6 route flush table 152 >/dev/null 2>&1 || true
|
|
ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
|
|
ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152
|
|
'';
|
|
};
|
|
my.ip.tailscale = "100.102.224.95";
|
|
services.udev.extraRules = ''
|
|
ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
|
|
ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
|
|
ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
|
|
'';
|
|
boot.kernel.sysctl = {
|
|
"net.ipv4.ip_forward" = "1";
|
|
"net.ipv6.conf.default.forwarding" = "1";
|
|
"net.ipv6.conf.all.forwarding" = "1";
|
|
"net.ipv6.conf.en-virginmedia.accept_ra" = "2";
|
|
"net.ipv6.conf.en-ee.accept_ra" = "2";
|
|
};
|
|
networking.nat = {
|
|
enable = true;
|
|
internalInterfaces = ["en-general"];
|
|
externalInterface = "en-virginmedia";
|
|
extraCommands = ''
|
|
# NAT packets going over EE plain.
|
|
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
|
|
|
|
# SNAT packets we're sending over tunnels.
|
|
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
|
|
iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
|
|
'';
|
|
};
|
|
services.dhcpd4 = {
|
|
enable = true;
|
|
interfaces = ["en-general"];
|
|
authoritative = true;
|
|
extraConfig = ''
|
|
subnet 192.168.1.0 netmask 255.255.255.0 {
|
|
option subnet-mask 255.255.255.0;
|
|
option routers 192.168.1.1;
|
|
option domain-name-servers 192.168.1.1;
|
|
option domain-name "house.as205479.net";
|
|
default-lease-time 600;
|
|
max-lease-time 3600;
|
|
option interface-mtu 1420; # Wireguard
|
|
|
|
range 192.168.1.100 192.168.1.200;
|
|
}
|
|
'';
|
|
machines = [
|
|
{
|
|
hostName = "totoro";
|
|
ethernetAddress = "40:8d:5c:1f:e8:68";
|
|
ipAddress = "192.168.1.40";
|
|
}
|
|
{
|
|
hostName = "totoro-pfsense";
|
|
ethernetAddress = "52:54:00:cf:cd:94";
|
|
ipAddress = "192.168.1.41";
|
|
}
|
|
{
|
|
hostName = "kvm";
|
|
ethernetAddress = "00:0d:5d:1b:14:ba";
|
|
ipAddress = "192.168.1.50";
|
|
}
|
|
{
|
|
hostName = "printer-xerox";
|
|
ethernetAddress = "9c:93:4e:ad:1f:7b";
|
|
ipAddress = "192.168.1.51";
|
|
}
|
|
];
|
|
};
|
|
networking.wireguard = let
|
|
ifBase = {
|
|
listenPort = null;
|
|
allowedIPsAsRoutes = false;
|
|
};
|
|
peerBase = {
|
|
allowedIPs = [
|
|
"0.0.0.0/0"
|
|
"::/0"
|
|
];
|
|
};
|
|
in {
|
|
enable = true;
|
|
interfaces.wg-tuvok-vm = ifBase // {
|
|
ips = [
|
|
"2a09:a442::1:1/112"
|
|
"92.118.30.0/31"
|
|
];
|
|
listenPort = 51820;
|
|
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
|
peers = [(peerBase // {
|
|
endpoint = "92.118.28.252:51820";
|
|
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
|
})];
|
|
postSetup = ''
|
|
wg set wg-tuvok-vm fwmark 0xbeef
|
|
'';
|
|
};
|
|
interfaces.wg-tuvok-ee = ifBase // {
|
|
ips = [
|
|
"2a09:a442::2:1/112"
|
|
"92.118.30.2/31"
|
|
];
|
|
listenPort = 51821;
|
|
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
|
peers = [(peerBase // {
|
|
endpoint = "[2a09:a441::f00f]:51821";
|
|
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
|
})];
|
|
postSetup = ''
|
|
wg set wg-tuvok-ee fwmark 0xdead
|
|
'';
|
|
};
|
|
};
|
|
|
|
services.unifi = {
|
|
enable = true;
|
|
openPorts = false;
|
|
unifiPackage = depot.pkgs.unifi;
|
|
};
|
|
services.prometheus.exporters.unifi-poller = {
|
|
enable = true;
|
|
controllers = [{
|
|
url = "https://localhost:8443";
|
|
verify_ssl = false;
|
|
user = "unifipoller";
|
|
pass = pkgs.writeTextFile { name = "unifipoller-password"; text = "unifipoller"; };
|
|
}];
|
|
};
|
|
|
|
networking.firewall = {
|
|
interfaces.en-general = {
|
|
allowedTCPPorts = [
|
|
8080 6789 # Unifi
|
|
53 # DNS
|
|
];
|
|
allowedUDPPorts = [
|
|
3478 10001 # Unifi
|
|
53 # DNS
|
|
];
|
|
};
|
|
interfaces.wg-tuvok-ee = {
|
|
allowedUDPPorts = [
|
|
3784 # BFD
|
|
];
|
|
};
|
|
interfaces.wg-tuvok-vm = {
|
|
allowedUDPPorts = [
|
|
3784 # BFD
|
|
];
|
|
};
|
|
extraCommands = ''
|
|
iptables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
|
|
ip6tables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360
|
|
'';
|
|
};
|
|
|
|
services.ddclient = {
|
|
enable = false;
|
|
protocol = "cloudflare";
|
|
domains = ["home.lukegb.com"];
|
|
zone = "lukegb.com";
|
|
password = secrets.cloudflareCredentials.token;
|
|
use = "if";
|
|
extraConfig = ''
|
|
if=en-virginmedia
|
|
daemon=0
|
|
'';
|
|
};
|
|
systemd.services.ddclient.serviceConfig.ExecStart = let
|
|
ddclient = pkgs.perlPackages.buildPerlPackage rec {
|
|
pname = "ddclient";
|
|
version = "3.9.1";
|
|
src = pkgs.fetchFromGitHub {
|
|
owner = "ddclient";
|
|
repo = "ddclient";
|
|
rev = "11a583b003920f8e15591813598b70061d1a4654";
|
|
sha256 = "sha256:1xz09vkii3mc2jmfwx9is07i06iiryv51571vdnl4m5mdnvsmlwb";
|
|
};
|
|
outputs = [ "out" ];
|
|
doCheck = false;
|
|
buildInputs = with pkgs.perlPackages; [ IOSocketSSL DigestSHA1 DataValidateIP JSONPP ];
|
|
nativeBuildInputs = with pkgs; [ autoreconfHook makeWrapper ];
|
|
preConfigure = ''
|
|
touch Makefile.PL
|
|
'';
|
|
postInstall = ''
|
|
patchShebangs $out/bin/ddclient
|
|
wrapProgram $out/bin/ddclient \
|
|
--suffix PATH : ${lib.makeBinPath (with pkgs; [ pkgs.iproute ])} \
|
|
--prefix PERL5LIB : $PERL5LIB
|
|
'';
|
|
};
|
|
RuntimeDirectory = "ddclient";
|
|
in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf";
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
ethtool
|
|
];
|
|
|
|
services.coredns = {
|
|
enable = true;
|
|
config = ''
|
|
.:53 {
|
|
bind 192.168.1.1 127.0.0.53 2a09:a443::1 2a09:a443:1::1
|
|
acl {
|
|
allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32
|
|
block
|
|
}
|
|
hosts /dev/null {
|
|
216.239.38.120 stadia.google.com stadia.com
|
|
fallthrough
|
|
}
|
|
loadbalance
|
|
forward . tls://8.8.8.8 tls://8.8.4.4 {
|
|
tls_servername dns.google
|
|
}
|
|
cache {
|
|
success 4096
|
|
denial 1024
|
|
prefetch 512
|
|
}
|
|
prometheus :9153
|
|
errors
|
|
log
|
|
}
|
|
'';
|
|
};
|
|
my.prometheus.additionalExporterPorts.coredns = 9153;
|
|
networking.resolvconf.extraConfig = ''
|
|
name_servers='127.0.0.53'
|
|
'';
|
|
|
|
services.prometheus.exporters.smokeping = {
|
|
enable = true;
|
|
hosts = [
|
|
"8.8.8.8" # Google Public DNS
|
|
"2001:4860:4860::8888"
|
|
"youtube.com" "ads.google.com" "google.com"
|
|
|
|
"1.1.1.1" # Cloudflare DNS
|
|
"2606:4700:4700::1111"
|
|
|
|
"twitter.com"
|
|
|
|
"store.steampowered.com"
|
|
"api.steampowered.com"
|
|
|
|
"prod.euw1.lol.riotgames.com" # League of Legends EUW
|
|
|
|
"eu.battle.net"
|
|
"185.60.112.157" "185.60.112.158" # Diablo 3/HotS/Hearthstone
|
|
"185.60.114.159" # Overwatch
|
|
|
|
# BFOB TS
|
|
"2a01:a500:85:3::2"
|
|
"37.9.61.53"
|
|
];
|
|
};
|
|
|
|
services.bird2 = {
|
|
enable = true;
|
|
config = ''
|
|
router id 92.118.30.254;
|
|
|
|
protocol kernel {
|
|
kernel table 150;
|
|
metric 0;
|
|
ipv4 {
|
|
import none;
|
|
export all;
|
|
};
|
|
};
|
|
protocol kernel {
|
|
kernel table 150;
|
|
metric 0;
|
|
ipv6 {
|
|
import none;
|
|
export all;
|
|
};
|
|
};
|
|
protocol device {};
|
|
|
|
protocol static export4 {
|
|
ipv4 {};
|
|
route 0.0.0.0/0 via 92.118.30.1 bfd {
|
|
# Virgin Media
|
|
preference = 100;
|
|
};
|
|
route 0.0.0.0/0 via 92.118.30.3 bfd {
|
|
# EE
|
|
preference = 10;
|
|
};
|
|
};
|
|
protocol static export6 {
|
|
ipv6 {};
|
|
route ::/0 via 2a09:a442::1:2 bfd {
|
|
# Virgin Media
|
|
preference = 100;
|
|
};
|
|
route ::/0 via 2a09:a442::2:2 bfd {
|
|
# EE
|
|
preference = 10;
|
|
};
|
|
|
|
# Covering route...
|
|
route 2a09:a443::/64 via "en-general";
|
|
route 2a09:a443::/32 unreachable;
|
|
};
|
|
|
|
protocol bfd {
|
|
interface "*" {
|
|
min rx interval 10ms;
|
|
min tx interval 50ms;
|
|
idle tx interval 1s;
|
|
multiplier 20;
|
|
};
|
|
neighbor 92.118.30.1;
|
|
neighbor 2a09:a442::1:2;
|
|
neighbor 92.118.30.3;
|
|
neighbor 2a09:a442::2:2;
|
|
};
|
|
'';
|
|
};
|
|
|
|
services.radvd = {
|
|
enable = true;
|
|
config = ''
|
|
interface en-general {
|
|
AdvSendAdvert on;
|
|
AdvLinkMTU 1420; # Wireguard
|
|
AdvManagedFlag on;
|
|
|
|
RDNSS 2a09:a443::1 {};
|
|
DNSSL house.as205479.net {};
|
|
|
|
prefix 2a09:a443::/64 {
|
|
AdvOnLink on;
|
|
AdvAutonomous on;
|
|
};
|
|
prefix 2a09:a443:1::/48 {
|
|
AdvOnLink on;
|
|
AdvAutonomous off;
|
|
};
|
|
};
|
|
'';
|
|
};
|
|
services.dhcpd6 = {
|
|
enable = true;
|
|
interfaces = ["en-general"];
|
|
authoritative = true;
|
|
extraConfig = ''
|
|
subnet6 2a09:a443:1::/48 {
|
|
range6 2a09:a443:1:1::/64;
|
|
range6 2a09:a443:1:2::/64 temporary;
|
|
prefix6 2a09:a443:1:1000:: 2a09:a443:1:ffff:: /56;
|
|
|
|
option dhcp6.name-servers 2a09:a443:1::1;
|
|
option dhcp6.domain-search "house.as205479.net";
|
|
}
|
|
'';
|
|
};
|
|
|
|
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
|
|
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
|
|
-web.listen-address 0.0.0.0:9324 \
|
|
-bird.socket /var/run/bird.ctl \
|
|
-bird.v2=true \
|
|
-format.new=true
|
|
'';
|
|
|
|
systemd.services.ee-scrape-data = let
|
|
scriptFile = ./ee-scrape-data.py;
|
|
python = pkgs.python3.withPackages (pm: with pm; [
|
|
requests
|
|
beautifulsoup4
|
|
html5lib
|
|
]);
|
|
in {
|
|
enable = true;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${python}/bin/python ${scriptFile} /run/prometheus-textfile-exports/ee-scrape-data.prom";
|
|
};
|
|
};
|
|
systemd.timers.ee-scrape-data = {
|
|
enable = true;
|
|
wantedBy = [ "multi-user.target" ];
|
|
timerConfig = {
|
|
OnBootSec = "2m";
|
|
OnUnitInactiveSec = "1m";
|
|
RandomizedDelaySec = "20";
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "21.03";
|
|
}
|