From 7134fe904adce4d74025d0a1107429deb282f348 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Mon, 30 Aug 2021 19:58:21 +0100 Subject: [PATCH] ops/nixos: implement BFD+WG tunneling for mldn-rd --- ops/nixos/blade-tuvok/default.nix | 136 +++++++ ops/nixos/lib/bgp.nix | 18 + ops/nixos/lib/blade-router.nix | 2 +- ops/nixos/lib/blade.nix | 24 +- ops/nixos/lib/bvm.nix | 10 +- .../coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa | 7 +- .../coredns/zones/db.28.118.92.in-addr.arpa | 4 +- .../coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa | 4 +- .../coredns/zones/db.30.118.92.in-addr.arpa | 12 +- ops/nixos/lib/coredns/zones/db.as205479.net | 16 +- ops/nixos/lib/switch-prebuilt.nix | 2 +- ops/nixos/swann/README.md | 4 +- ops/nixos/swann/default.nix | 343 ++++++++++++++---- 13 files changed, 483 insertions(+), 99 deletions(-) diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix index f8d63e4dec..1f97669f9d 100644 --- a/ops/nixos/blade-tuvok/default.nix +++ b/ops/nixos/blade-tuvok/default.nix @@ -19,6 +19,24 @@ in { hostName = "blade-tuvok"; hostId = "525229f7"; firewall.allowedTCPPorts = [ 80 443 ]; + firewall.allowedUDPPorts = [ + # Wireguard + 51820 51821 + ]; + localCommands = '' + # Check if we already have our little minicontainer setup + ip netns list | grep -q wg-endpoint || ( + ip netns add wg-endpoint + ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint + ip link set dev ve-wg-endpoint master br-public + ip link set dev ve-wg-endpoint up + ip -n wg-endpoint link set dev ve-int up + ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int + ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int + ip -n wg-endpoint route add default via 92.118.28.1 + ip -n wg-endpoint route add default via 2a09:a441::1 + ) + ''; }; my.ip.tailscale = "100.119.123.33"; my.blade.bay = 6; @@ -88,4 +106,122 @@ in { vrrp.priority = 100; }; + + networking.wireguard = let + ifBase = { + listenPort = null; + allowedIPsAsRoutes = false; + socketNamespace = "wg-endpoint"; + }; + peerBase.allowedIPs = [ + "0.0.0.0/0" + "::/0" + ]; + swannBase = ifBase // { + privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey; + peers = [(peerBase // { + endpoint = null; # dynamic + publicKey = secrets.wireguard.tuvok-swann.swann.publicKey; + })]; + }; + in { + enable = true; + interfaces.wg-swann-vm = swannBase // { + ips = [ + "2a09:a442::1:2/112" + "92.118.30.1/31" + ]; + listenPort = 51820; + }; + interfaces.wg-swann-ee = swannBase // { + ips = [ + "2a09:a442::2:2/112" + "92.118.30.3/31" + ]; + listenPort = 51821; + }; + }; + environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile { + name = "bird-wg-endpoint.conf"; + text = '' + router id 92.118.28.252; + protocol kernel { + persist; + ipv4 { + import none; + export all; + }; + }; + protocol kernel { + persist; + ipv6 { + import none; + export all; + }; + }; + protocol device {}; + + protocol static export4 { + ipv4 {}; + route 92.118.30.0/24 + via 92.118.30.0 weight 1 bfd # Virgin Media + via 92.118.30.2 weight 2 bfd; # EE + }; + protocol static export6 { + ipv6 {}; + route 2a09:a443::/32 + via 2a09:a442::1:1 weight 1 bfd # Virgin Media + via 2a09:a442::2:1 weight 2 bfd; # EE + }; + + protocol bfd { + interface "*" { + min rx interval 10ms; + min tx interval 50ms; + idle tx interval 1s; + multiplier 20; + }; + neighbor 92.118.30.0; + neighbor 2a09:a442::1:1; + neighbor 92.118.30.2; + neighbor 2a09:a442::2:1; + }; + ''; + checkPhase = '' + ${pkgs.bird2}/bin/bird -d -p -c $out + ''; + }; + systemd.services.bird-wg-endpoint = { + wantedBy = [ "multi-user.target" ]; + reloadIfChanged = true; + + description = "BIRD inside wg-endpoint netns"; + after = [ "network.target" ]; + restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ]; + + serviceConfig = { + Type = "forking"; + Restart = "on-failure"; + CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID" + # see bird/sysdep/linux/syspriv.h + "CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ]; + ProtectSystem = "full"; + ProtectHome = "yes"; + SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io"; + MemoryDenyWriteExecute = "yes"; + + ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down"; + ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl"; + ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'"; + NetworkNamespacePath = "/var/run/netns/wg-endpoint"; + }; + }; + services.lukegbgp.config.export = { + v4Extra = '' + route 92.118.30.0/24 via 92.118.28.252; + ''; + v6Extra = '' + route 2a09:a443::/32 via 2a09:a441::f00f; + ''; + }; } diff --git a/ops/nixos/lib/bgp.nix b/ops/nixos/lib/bgp.nix index 5fd7647c08..198ca9edc0 100644 --- a/ops/nixos/lib/bgp.nix +++ b/ops/nixos/lib/bgp.nix @@ -177,13 +177,25 @@ in { type = listOf str; default = ["92.118.31.0/24"]; }; + v4Extra = mkOption { #lukegbgp.config.export.v4Extra + type = lines; + default = ""; + }; v6 = mkOption { # lukegbgp.config.export.v6 type = listOf str; default = ["2a09:a440::/48"]; }; + v6Extra = mkOption { #lukegbgp.config.export.v6Extra + type = lines; + default = ""; + }; }; }; }; + bfd = mkOption { # lukegbgp.config.bfd + type = lines; + default = ""; + }; }; }; }; @@ -287,6 +299,7 @@ in { }; }; ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v4} + ${config.services.lukegbgp.config.export.v4Extra} }; protocol static export6 { ipv6 { @@ -315,6 +328,11 @@ in { }; }; ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.export.v6} + ${config.services.lukegbgp.config.export.v6Extra} + }; + + protocol bfd { + ${config.services.lukegbgp.config.bfd} }; ''; }; diff --git a/ops/nixos/lib/blade-router.nix b/ops/nixos/lib/blade-router.nix index 50b3eaa3d4..940dee485b 100644 --- a/ops/nixos/lib/blade-router.nix +++ b/ops/nixos/lib/blade-router.nix @@ -280,7 +280,7 @@ in } ''; }; - + services.radvd = { enable = true; config = '' diff --git a/ops/nixos/lib/blade.nix b/ops/nixos/lib/blade.nix index de7be74455..12f5bdcb03 100644 --- a/ops/nixos/lib/blade.nix +++ b/ops/nixos/lib/blade.nix @@ -44,7 +44,7 @@ in { ]; my.rundeck.tags = [ "blade" ]; - + fileSystems = let zfs = device: { device = device; @@ -73,10 +73,10 @@ in { fsType = "xfs"; }; }); - + boot.loader.grub.enable = true; boot.loader.grub.version = 2; - + # Networking! networking = { domain = "blade.as205479.net"; @@ -105,9 +105,9 @@ in { address = "10.100.2.${toString (100 + config.my.blade.bay)}"; prefixLength = 24; }]; - + defaultGateway = lib.mkDefault "10.100.0.1"; - + firewall.allowedUDPPorts = [ 41641 # Tailscale ]; @@ -128,14 +128,14 @@ in { '') + (lib.optionalString (config.my.blade.macAddress.internet != null) '' ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet" ''); - + virtualisation.podman.enable = true; - + environment.systemPackages = with pkgs; [ ceph xfsprogs ]; - + services.ceph = { enable = true; global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; @@ -158,15 +158,15 @@ in { description = "Ceph OSD pre-start"; before = [ "network-online.target" "ceph-osd.target" ]; wantedBy = [ "ceph-osd.target" ]; - + path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; - + serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; }; }; - + virtualisation.libvirtd = { enable = true; qemuRunAsRoot = false; @@ -187,7 +187,7 @@ in { Storage=none ProcessSizeMax=0 ''; - + system.stateVersion = "21.05"; }; } diff --git a/ops/nixos/lib/bvm.nix b/ops/nixos/lib/bvm.nix index 3ea0845104..5ec3320598 100644 --- a/ops/nixos/lib/bvm.nix +++ b/ops/nixos/lib/bvm.nix @@ -27,7 +27,7 @@ ]; powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; - + fileSystems = { "/" = { device = "/dev/vda1"; @@ -38,7 +38,7 @@ fsType = "vfat"; }; }; - + # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; @@ -46,7 +46,7 @@ nix.maxJobs = lib.mkDefault 2; my.rundeck.tags = [ "bvm" ]; - + # Networking! networking = { domain = "blade.as205479.net"; @@ -60,12 +60,12 @@ address = "10.100.0.1"; interface = "enp1s0"; }; - + firewall.allowedUDPPorts = [ 41641 # Tailscale ]; }; - + services.qemuGuest.enable = true; }; } diff --git a/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa b/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa index 12d6173ccc..0ddb42aa7e 100644 --- a/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa +++ b/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa @@ -3,6 +3,11 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300 $INCLUDE tmpl.ns + +1.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.vm-tuvok.mldn-rd.as205479.net. +2.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net. +1.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net. +2.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.28.118.92.in-addr.arpa b/ops/nixos/lib/coredns/zones/db.28.118.92.in-addr.arpa index 734ee82136..5bccad47cd 100644 --- a/ops/nixos/lib/coredns/zones/db.28.118.92.in-addr.arpa +++ b/ops/nixos/lib/coredns/zones/db.28.118.92.in-addr.arpa @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 12 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 13 600 450 3600 300 $INCLUDE tmpl.ns @@ -259,7 +259,7 @@ $INCLUDE tmpl.ns 249 600 IN PTR 92-118-28-249.ptr.as205479.net. 250 600 IN PTR 92-118-28-250.ptr.as205479.net. 251 600 IN PTR 92-118-28-251.ptr.as205479.net. -252 600 IN PTR 92-118-28-252.ptr.as205479.net. +252 600 IN PTR wg-gw.public.as205479.net. 253 600 IN PTR blade-paris.public.as205479.net. 254 600 IN PTR blade-tuvok.public.as205479.net. 255 600 IN PTR 92-118-28-255.ptr.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa b/ops/nixos/lib/coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa index 12d6173ccc..c449abc62a 100644 --- a/ops/nixos/lib/coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa +++ b/ops/nixos/lib/coredns/zones/db.3.4.4.a.9.0.a.2.ip6.arpa @@ -3,6 +3,8 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300 $INCLUDE tmpl.ns + +1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn-rd.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa b/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa index f2718676c1..ded5b2df67 100644 --- a/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa +++ b/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa @@ -3,14 +3,14 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 1 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300 $INCLUDE tmpl.ns -0 600 IN PTR 92-118-30-0.ptr.as205479.net. -1 600 IN PTR 92-118-30-1.ptr.as205479.net. -2 600 IN PTR 92-118-30-2.ptr.as205479.net. -3 600 IN PTR 92-118-30-3.ptr.as205479.net. +0 600 IN PTR mldn.vm-tuvok.mldn-rd.as205479.net. +1 600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net. +2 600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net. +3 600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net. 4 600 IN PTR 92-118-30-4.ptr.as205479.net. 5 600 IN PTR 92-118-30-5.ptr.as205479.net. 6 600 IN PTR 92-118-30-6.ptr.as205479.net. @@ -261,5 +261,5 @@ $INCLUDE tmpl.ns 251 600 IN PTR 92-118-30-251.ptr.as205479.net. 252 600 IN PTR 92-118-30-252.ptr.as205479.net. 253 600 IN PTR 92-118-30-253.ptr.as205479.net. -254 600 IN PTR 92-118-30-254.ptr.as205479.net. +254 600 IN PTR mldn-rd.as205479.net. 255 600 IN PTR 92-118-30-255.ptr.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.as205479.net b/ops/nixos/lib/coredns/zones/db.as205479.net index 3f513b86a8..b0765ec9cb 100644 --- a/ops/nixos/lib/coredns/zones/db.as205479.net +++ b/ops/nixos/lib/coredns/zones/db.as205479.net @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 26 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 27 600 450 3600 300 ; NB: this are also glue records in Google Domains. $INCLUDE tmpl.ns @@ -75,6 +75,18 @@ fp-la.int 3600 IN A 208.77.235.186 fp-la-sec.int 3600 IN A 206.214.42.86 fp-la.int 3600 IN A 206.214.42.86 +; mldn rd +mldn.vm-tuvok.mldn-rd 3600 IN A 92.118.30.0 +mldn.vm-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::1:1 +tuvok.vm-tuvok.mldn-rd 3600 IN A 92.118.30.1 +tuvok.vm-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::1:2 +mldn.ee-tuvok.mldn-rd 3600 IN A 92.118.30.2 +mldn.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:1 +tuvok.ee-tuvok.mldn-rd 3600 IN A 92.118.30.3 +tuvok.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:2 +mldn-rd 3600 IN A 92.118.30.254 +mldn-rd 3600 IN AAAA 2a09:a443::1 + ; blade internal blade-oa.blade 3600 IN A 10.100.1.200 blade-vcenet1.blade 3600 IN A 10.100.1.201 @@ -125,6 +137,8 @@ _ceph-mon._tcp.storage.blade 60 IN SRV 10 10 6789 blade-paris.storage.blade.a ; public gw.public 3600 IN A 92.118.28.1 gw.public 3600 IN AAAA 2a09:a441::1 +wg-gw.public 3600 IN A 92.118.28.252 +wg-gw.public 3600 IN AAAA 2a09:a441::f00f blade-tuvok.public 3600 IN A 92.118.28.254 blade-tuvok.public 3600 IN AAAA 2a09:a441::ffff blade-paris.public 3600 IN A 92.118.28.253 diff --git a/ops/nixos/lib/switch-prebuilt.nix b/ops/nixos/lib/switch-prebuilt.nix index 92c54643cc..4b27382c06 100644 --- a/ops/nixos/lib/switch-prebuilt.nix +++ b/ops/nixos/lib/switch-prebuilt.nix @@ -13,7 +13,7 @@ pkgs.writeShellScriptBin "switch-prebuilt" '' if [[ "$system" == "latest" ]]; then tmpdir="$(mktemp -d)" trap '{ rm -rf -- "$tmpdir"; }' EXIT - + ${pkgs.curl}/bin/curl -so "$tmpdir/archive.zip" 'https://hg.lukegb.com/api/v4/projects/lukegb%2Fdepot/jobs/artifacts/branch%2Fdefault/download?job=nixCache' ${pkgs.unzip}/bin/unzip -d "$tmpdir" -q -o "$tmpdir/archive.zip" system="$(${pkgs.jq}/bin/jq -r ".\"$(hostname)\"" "$tmpdir/systems.json")" diff --git a/ops/nixos/swann/README.md b/ops/nixos/swann/README.md index 2af2cc2d1f..6004f75956 100644 --- a/ops/nixos/swann/README.md +++ b/ops/nixos/swann/README.md @@ -14,5 +14,5 @@ Hardware running NixOS in my flat. NICs on: -* `ens-virginmedia` Virgin Media (DHCP) -* `ens-general` General (192.168.1.1) +* `en-virginmedia` Virgin Media (DHCP) +* `en-general` General (192.168.1.1) diff --git a/ops/nixos/swann/default.nix b/ops/nixos/swann/default.nix index c8e48d7317..12e6f82526 100644 --- a/ops/nixos/swann/default.nix +++ b/ops/nixos/swann/default.nix @@ -6,6 +6,11 @@ let inherit (depot.ops) secrets; in { + imports = [ + # We include this just so it sets some sysctls and firewall settings. + ../lib/bgp.nix + ]; + boot.initrd.availableKernelModules = [ "sd_mod" "ahci" @@ -33,57 +38,144 @@ in { # Networking! networking = { + # Routing tables: + # bgp (150) -- contains default routes over WG tunnels + # ee (201) -- table contains a static default route via EE + # main (254) -- DHCP routes (aka VM) + # Conventional lookup order is bgp, main. ee has routing rules. + hostName = "swann"; # Define your hostname. domain = "int.as205479.net"; nameservers = ["8.8.8.8" "8.8.4.4"]; useDHCP = false; interfaces = { - ens-virginmedia = { + lo = { + ipv4.addresses = [ + { address = "127.0.0.1"; prefixLength = 8; } + { address = "92.118.30.254"; prefixLength = 32; } + ]; + }; + en-virginmedia = { useDHCP = true; macAddress = "e4:3a:6e:16:07:61"; }; - ens-general = { + en-ee = { + ipv4.addresses = [ + { address = "192.168.200.2"; prefixLength = 24; } + ]; + ipv4.routes = [{ + via = "192.168.200.1"; + address = "0.0.0.0"; + prefixLength = 0; + options.table = "201"; + }]; + }; + en-general = { ipv4.addresses = [ { address = "192.168.1.1"; prefixLength = 23; } ]; ipv6.addresses = [ - { address = "2a02:88fd:f:d::2"; prefixLength = 64; } - ]; - ipv6.routes = [ - { address = "2a02:88fd:f:d::"; prefixLength = 64; options.metric = "100"; } + { address = "2a09:a443::1"; prefixLength = 64; } + { address = "2a09:a443:1::1"; prefixLength = 48; } ]; }; }; + + dhcpcd.extraConfig = '' + interface en-virginmedia + metric 100 + + interface en-ee + metric 250 + ''; + localCommands = let + claimedPriorities = { min = 10000; max = 10100; }; + rules = [ + # Route traffic to EE via WG... via EE. + { priority = 10000; both = "fwmark 0xdead table 201"; } + # Route traffic to VM via WG... via VM DHCP in main table. + { priority = 10001; both = "fwmark 0xbeef table main"; } + + # Make ping work over the tunnels. + { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; } + { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; } + + # Now some subset of RFC1918 via main table too. + { priority = 10020; v4 = "to 192.168.0.0/16 table main"; } + { priority = 10021; v4 = "to 10.0.0.0/8 table main"; } + { priority = 10022; v4 = "to 172.16.0.0/12 table main"; } + + # And the linknets... + { priority = 10023; v4 = "to 92.118.30.0/24 table main"; } + { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; } + { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; } + + # Everything else over WG. + { priority = 10030; both = "table 150"; } + ]; + clearRules = map (x: '' + ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true + ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true + '') (lib.range claimedPriorities.min (claimedPriorities.max - 1)); + ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }: + assert (both == "" || (v4 == "" && v6 == "")); + assert priority >= claimedPriorities.min; + assert priority < claimedPriorities.max; + let + rv4 = if v4 != "" then v4 else both; + rv6 = if v6 != "" then v6 else both; + in '' + ${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""} + ${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""} + ''; + addRules = map ruleToLine rules; + in '' + ${lib.concatStringsSep "\n" clearRules} + ${lib.concatStringsSep "\n" addRules} + + ip -4 route flush table 151 >/dev/null 2>&1 || true + ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151 + ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151 + ip -6 route flush table 151 >/dev/null 2>&1 || true + ip -6 route add 2a09:a442::1:0/112 dev wg-tuvok-vm table 151 + ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151 + + ip -4 route flush table 152 >/dev/null 2>&1 || true + ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151 + ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152 + ip -6 route flush table 152 >/dev/null 2>&1 || true + ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152 + ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152 + ''; }; my.ip.tailscale = "100.102.224.95"; services.udev.extraRules = '' - ATTR{address}=="e4:3a:6e:16:07:62", NAME="ens-virginmedia" - ATTR{address}=="e4:3a:6e:16:07:67", NAME="ens-general" + ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia" + ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee" + ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general" ''; boot.kernel.sysctl = { "net.ipv4.ip_forward" = "1"; "net.ipv6.conf.default.forwarding" = "1"; "net.ipv6.conf.all.forwarding" = "1"; - "net.ipv6.conf.ens-virginmedia.accept_ra" = "2"; + "net.ipv6.conf.en-virginmedia.accept_ra" = "2"; }; networking.nat = { enable = true; - externalInterface = "ens-virginmedia"; - internalInterfaces = ["ens-general"]; - forwardPorts = [ - { destination = "192.168.1.40:22"; proto = "tcp"; sourcePort = 10022; } - { destination = "192.168.1.40:41641"; proto = "udp"; sourcePort = 41641; } - { destination = "192.168.1.40:80"; proto = "tcp"; sourcePort = 80; } - { destination = "192.168.1.40:443"; proto = "tcp"; sourcePort = 443; } + internalInterfaces = ["en-general"]; + externalInterface = "en-virginmedia"; + extraCommands = '' + # NAT packets going over EE plain. + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE - # IPFS - { destination = "192.168.1.40:4001"; proto = "tcp"; sourcePort = 4001; } - { destination = "192.168.1.40:4001"; proto = "udp"; sourcePort = 4001; } - ]; + # SNAT packets we're sending over tunnels. + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254 + ''; }; services.dhcpd4 = { enable = true; - interfaces = ["ens-general"]; + interfaces = ["en-general"]; authoritative = true; extraConfig = '' subnet 192.168.1.0 netmask 255.255.255.0 { @@ -93,6 +185,7 @@ in { option domain-name "house.as205479.net"; default-lease-time 600; max-lease-time 3600; + option interface-mtu 1420; # Wireguard range 192.168.1.100 192.168.1.200; } @@ -120,18 +213,50 @@ in { } ]; }; - networking.localCommands = '' - tc qdisc del dev ens-virginmedia root || true - tc qdisc add dev ens-virginmedia root cake bandwidth 20Mbit docsis nat dual-srchost - - ip link add name ifb-virginmedia type ifb || true - tc qdisc del dev ens-virginmedia ingress || true - tc qdisc add dev ens-virginmedia handle ffff: ingress - tc qdisc del dev ifb-virginmedia root || true - tc qdisc add dev ifb-virginmedia root cake bandwidth 450Mbit besteffort docsis nat wash dual-dsthost - ip link set dev ifb-virginmedia up - tc filter add dev ens-virginmedia parent ffff: matchall action mirred egress redirect dev ifb-virginmedia - ''; + networking.wireguard = let + ifBase = { + listenPort = null; + allowedIPsAsRoutes = false; + }; + peerBase = { + allowedIPs = [ + "0.0.0.0/0" + "::/0" + ]; + }; + in { + enable = true; + interfaces.wg-tuvok-vm = ifBase // { + ips = [ + "2a09:a442::1:1/112" + "92.118.30.0/31" + ]; + listenPort = 51820; + privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; + peers = [(peerBase // { + endpoint = "92.118.28.252:51820"; + publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; + })]; + postSetup = '' + wg set wg-tuvok-vm fwmark 0xbeef + ''; + }; + interfaces.wg-tuvok-ee = ifBase // { + ips = [ + "2a09:a442::2:1/112" + "92.118.30.2/31" + ]; + listenPort = 51821; + privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; + peers = [(peerBase // { + endpoint = "92.118.28.252:51821"; + publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; + })]; + postSetup = '' + wg set wg-tuvok-ee fwmark 0xdead + ''; + }; + }; services.unifi = { enable = true; @@ -149,7 +274,7 @@ in { }; networking.firewall = { - interfaces.ens-general = { + interfaces.en-general = { allowedTCPPorts = [ 8080 6789 # Unifi 53 # DNS @@ -159,6 +284,20 @@ in { 53 # DNS ]; }; + interfaces.wg-tuvok-ee = { + allowedUDPPorts = [ + 3784 # BFD + ]; + }; + interfaces.wg-tuvok-vm = { + allowedUDPPorts = [ + 3784 # BFD + ]; + }; + extraCommands = '' + iptables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 + ip6tables -I FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 + ''; }; services.ddclient = { @@ -169,7 +308,7 @@ in { password = secrets.cloudflareCredentials.token; use = "if"; extraConfig = '' - if=ens-virginmedia + if=en-virginmedia daemon=0 ''; }; @@ -200,15 +339,17 @@ in { RuntimeDirectory = "ddclient"; in lib.mkForce "${lib.getBin ddclient}/bin/ddclient -file /run/${RuntimeDirectory}/ddclient.conf"; - environment.systemPackages = with pkgs; []; + environment.systemPackages = with pkgs; [ + ethtool + ]; services.coredns = { enable = true; config = '' .:53 { - bind 192.168.1.1 127.0.0.53 + bind 192.168.1.1 127.0.0.53 2a09:a443::1 2a09:a443:1::1 acl { - allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 + allow net 192.168.0.0/16 172.16.0.0/12 10.0.0.0/8 127.0.0.0/8 100.64.0.0/10 2a09:a443::/32 block } hosts /dev/null { @@ -261,38 +402,106 @@ in { ]; }; - # This is cursed. - services.ndppd = { - enable = false; - proxies.ens-virginmedia = { - router = false; - rules."2a02:88fd:f:d::/64" = { - method = "iface"; - interface = "ens-general"; - }; - }; - }; - networking.dhcpcd.extraConfig = '' - noipv6rs - - interface ens-virginmedia - ipv6rs - iaid 1 - ia_na 2 - ia_pd 3 ens-general/1/64 - ''; - services.radvd = { - enable = false; + services.bird2 = { + enable = true; config = '' - interface ens-general { - AdvSendAdvert on; - MaxRtrAdvInterval 60; - AdvDefaultLifetime 180; - prefix 2a02:88fd:f:d::/64 { - AdvValidLifetime 7200; - AdvPreferredLifetime 3600; + router id 92.118.30.254; + + protocol kernel { + kernel table 150; + metric 0; + ipv4 { + import none; + export all; }; }; + protocol kernel { + kernel table 150; + metric 0; + ipv6 { + import none; + export all; + }; + }; + protocol device {}; + + protocol static export4 { + ipv4 {}; + route 0.0.0.0/0 via 92.118.30.1 bfd { + # Virgin Media + preference = 100; + }; + route 0.0.0.0/0 via 92.118.30.3 bfd { + # EE + preference = 10; + }; + }; + protocol static export6 { + ipv6 {}; + route ::/0 via 2a09:a442::1:2 bfd { + # Virgin Media + preference = 100; + }; + route ::/0 via 2a09:a442::2:2 bfd { + # EE + preference = 10; + }; + + # Covering route... + route 2a09:a443::/64 via "en-general"; + route 2a09:a443::/32 unreachable; + }; + + protocol bfd { + interface "*" { + min rx interval 10ms; + min tx interval 50ms; + idle tx interval 1s; + multiplier 20; + }; + neighbor 92.118.30.1; + neighbor 2a09:a442::1:2; + neighbor 92.118.30.3; + neighbor 2a09:a442::2:2; + }; + ''; + }; + + services.radvd = { + enable = true; + config = '' + interface en-general { + AdvSendAdvert on; + AdvLinkMTU 1420; # Wireguard + AdvManagedFlag on; + + RDNSS 2a09:a443::1 {}; + DNSSL house.as205479.net {}; + + prefix 2a09:a443::/64 { + AdvOnLink on; + AdvAutonomous on; + }; + prefix 2a09:a443:1::/48 { + AdvOnLink on; + AdvAutonomous off; + }; + }; + ''; + }; + services.dhcpd6 = { + enable = true; + interfaces = ["en-general"]; + authoritative = true; + extraConfig = '' + subnet6 2a09:a443:1::/48 { + range6 2a09:a443:1:1::/64; + range6 2a09:a443:1:2::/64 temporary; + prefix6 2a09:a443:1:1000:: 2a09:a443:1:ffff:: /56; + + option dhcp6.name-servers 2a09:a443:1::1; + option dhcp6.domain-search "house.as205479.net"; + } ''; };