From 9be6bcaf2dd80f4f70967cfbf7ccfe641ea6f72a Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Fri, 14 Jan 2022 19:42:06 +0000 Subject: [PATCH] ops/nixos: set up gnetwork link --- ops/nixos/blade-tuvok/default.nix | 31 ++++- .../coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa | 4 +- .../coredns/zones/db.30.118.92.in-addr.arpa | 6 +- ops/nixos/lib/coredns/zones/db.as205479.net | 6 +- ops/nixos/swann/default.nix | 126 ++++++++++++++---- 5 files changed, 136 insertions(+), 37 deletions(-) diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix index f333dd5183..7e6cecc1d1 100644 --- a/ops/nixos/blade-tuvok/default.nix +++ b/ops/nixos/blade-tuvok/default.nix @@ -21,7 +21,7 @@ in { firewall.allowedTCPPorts = [ 80 443 ]; firewall.allowedUDPPorts = [ # Wireguard - 51820 51821 + 51820 51821 51822 ]; localCommands = '' # Check if we already have our little minicontainer setup @@ -116,12 +116,13 @@ in { "0.0.0.0/0" "::/0" ]; + swannPeerBase = peerBase // { + endpoint = null; # dynamic + publicKey = secrets.wireguard.tuvok-swann.swann.publicKey; + }; swannBase = ifBase // { privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey; - peers = [(peerBase // { - endpoint = null; # dynamic - publicKey = secrets.wireguard.tuvok-swann.swann.publicKey; - })]; + peers = [swannPeerBase]; }; in { enable = true; @@ -139,6 +140,16 @@ in { ]; listenPort = 51821; }; + interfaces.wg-swann-gnet = swannBase // { + ips = [ + "2a09:a442::3:2/112" + "92.118.30.5/31" + ]; + listenPort = 51822; + peers = [(swannPeerBase // { + endpoint = "185.250.189.204:51822"; + })]; + }; }; environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile { name = "bird-wg-endpoint.conf"; @@ -170,6 +181,10 @@ in { # EE preference = 10; }; + route 92.118.30.0/24 via 92.118.30.4 bfd { + # GNetwork + preference = 200; + }; }; protocol static export6 { ipv6 {}; @@ -181,6 +196,10 @@ in { # EE preference = 10; }; + route 2a09:a443::/32 via 2a09:a442::3:1 bfd { + # GNetwork + preference = 200; + }; }; protocol bfd { @@ -194,6 +213,8 @@ in { neighbor 2a09:a442::1:1; neighbor 92.118.30.2; neighbor 2a09:a442::2:1; + neighbor 92.118.30.4; + neighbor 2a09:a442::3:1; }; ''; checkPhase = '' diff --git a/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa b/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa index 0ddb42aa7e..027aa96bb2 100644 --- a/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa +++ b/ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 2 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 3 600 450 3600 300 $INCLUDE tmpl.ns @@ -11,3 +11,5 @@ $INCLUDE tmpl.ns 2.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net. 1.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net. 2.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net. +1.0.0.0.3.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.gnet-tuvok.mldn-rd.as205479.net. +2.0.0.0.3.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.gnet-tuvok.mldn-rd.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa b/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa index d00d714791..e545eb8285 100644 --- a/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa +++ b/ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 3 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 4 600 450 3600 300 $INCLUDE tmpl.ns @@ -11,8 +11,8 @@ $INCLUDE tmpl.ns 1 600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net. 2 600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net. 3 600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net. -4 600 IN PTR 92-118-30-4.ptr.as205479.net. -5 600 IN PTR 92-118-30-5.ptr.as205479.net. +4 600 IN PTR mldn.gnet-tuvok.mldn-rd.as205479.net. +5 600 IN PTR tuvok.gnet-tuvok.mldn-rd.as205479.net. 6 600 IN PTR 92-118-30-6.ptr.as205479.net. 7 600 IN PTR 92-118-30-7.ptr.as205479.net. 8 600 IN PTR 92-118-30-8.ptr.as205479.net. diff --git a/ops/nixos/lib/coredns/zones/db.as205479.net b/ops/nixos/lib/coredns/zones/db.as205479.net index 3d61ac6fe4..eefaa1c135 100644 --- a/ops/nixos/lib/coredns/zones/db.as205479.net +++ b/ops/nixos/lib/coredns/zones/db.as205479.net @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 43 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 44 600 450 3600 300 ; NB: this are also glue records in Google Domains. $INCLUDE tmpl.ns @@ -105,6 +105,10 @@ mldn.ee-tuvok.mldn-rd 3600 IN A 92.118.30.2 mldn.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:1 tuvok.ee-tuvok.mldn-rd 3600 IN A 92.118.30.3 tuvok.ee-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::2:2 +mldn.gnet-tuvok.mldn-rd 3600 IN A 92.118.30.4 +mldn.gnet-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::3:1 +tuvok.gnet-tuvok.mldn-rd 3600 IN A 92.118.30.5 +tuvok.gnet-tuvok.mldn-rd 3600 IN AAAA 2a09:a442::3:2 mldn-rd 3600 IN A 92.118.30.254 mldn-rd 3600 IN AAAA 2a09:a443::1 eduroam.mldn-rd 3600 IN A 92.118.30.253 diff --git a/ops/nixos/swann/default.nix b/ops/nixos/swann/default.nix index 0d4f113667..3dc1e8442f 100644 --- a/ops/nixos/swann/default.nix +++ b/ops/nixos/swann/default.nix @@ -40,13 +40,18 @@ in { networking = { # Routing tables: # bgp (150) -- contains default routes over WG tunnels - # ee (201) -- table contains a static default route via EE - # main (254) -- DHCP routes (aka VM) - # Conventional lookup order is bgp, main. ee has routing rules. + # wg-vm (151) -- contains default routes over WG tunnels + # wg-ee (152) -- contains default routes over WG tunnels + # wg-gnet (153) -- contains default routes over WG tunnels + # ee (201) -- table contains a default route via EE + # vm (202) -- table contains a default route via VM + # gnet (203) -- table contains a default route via gnetwork + # main (254) -- basically empty hostName = "swann"; # Define your hostname. domain = "int.as205479.net"; nameservers = ["8.8.8.8" "8.8.4.4"]; + useNetworkd = true; interfaces = { lo = { ipv4.addresses = [ @@ -59,16 +64,13 @@ in { useDHCP = true; macAddress = "e4:3a:6e:16:07:61"; }; + en-gnet = { + useDHCP = true; + # Additional options configured in networkd. + }; en-ee = { - ipv4.addresses = [ - { address = "192.168.200.2"; prefixLength = 24; } - ]; - ipv4.routes = [{ - via = "192.168.200.1"; - address = "0.0.0.0"; - prefixLength = 0; - options.table = "201"; - }]; + useDHCP = true; + # Additional options configured in networkd. }; en-general = { ipv4.addresses = [ @@ -97,24 +99,20 @@ in { }; }; - dhcpcd.extraConfig = '' - interface en-virginmedia - metric 100 - - interface en-ee - metric 250 - ''; localCommands = let claimedPriorities = { min = 10000; max = 10100; }; rules = [ # Route traffic to EE via WG... via EE. { priority = 10000; both = "fwmark 0xdead table 201"; } - # Route traffic to VM via WG... via VM DHCP in main table. - { priority = 10001; both = "fwmark 0xbeef table main"; } + # Route traffic to VM via WG... via VM DHCP in table 202. + { priority = 10001; both = "fwmark 0xbeef table 202"; } + # Route traffic to GNetwork via WG... via DHCP in table 203. + { priority = 10002; both = "fwmark 0xcafe table 203"; } # Make ping work over the tunnels. { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; } { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; } + { priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; } # Now some subset of RFC1918 via main table too. { priority = 10020; v4 = "to 192.168.0.0/16 table main"; } @@ -125,9 +123,7 @@ in { { priority = 10023; v4 = "to 92.118.30.0/24 table main"; } { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; } { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; } - - # And the Google VIP I'm (ab)using for Stadia (see CoreDNS below). - { priority = 10030; v4 = "to 216.239.38.120/32 table main"; } + { priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; } # add-on.ee.co.uk goes via EE. { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; } @@ -136,7 +132,13 @@ in { { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; } # Everything else over WG. - { priority = 10099; both = "table 150"; } + { priority = 10080; both = "table 150"; } + + # Fallbacks via GNetwork, VM, EE + # Sometimes this seems to be required to get things moving, for some super unclear reason. + { priority = 10090; both = "table 203"; } + { priority = 10091; both = "table 202"; } + { priority = 10092; both = "table 201"; } ]; clearRules = map (x: '' ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true @@ -166,17 +168,33 @@ in { ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151 ip -4 route flush table 152 >/dev/null 2>&1 || true - ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151 + ip -4 route add 92.118.30.2/31 dev wg-tuvok-ee table 152 ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152 ip -6 route flush table 152 >/dev/null 2>&1 || true ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152 ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152 + + ip -4 route flush table 153 >/dev/null 2>&1 || true + ip -4 route add 92.118.30.4/31 dev wg-tuvok-gnet table 153 + ip -4 route add default via 92.118.30.5 dev wg-tuvok-gnet table 153 + ip -6 route flush table 153 >/dev/null 2>&1 || true + ip -6 route add 2a09:a442::3:0/112 dev wg-tuvok-gnet table 153 + ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153 ''; }; + systemd.network = { + enable = true; + networks."40-en-ee".dhcpV4Config.RouteTable = 201; + networks."40-en-ee".linkConfig.RequiredForOnline = "no"; + networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202; + networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no"; + networks."40-en-gnet".dhcpV4Config.RouteTable = 203; + }; my.ip.tailscale = "100.102.224.95"; services.udev.extraRules = '' ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia" ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee" + ATTR{address}=="e4:3a:6e:16:07:64", NAME="en-gnet" ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general" ''; boot.kernel.sysctl = { @@ -185,6 +203,7 @@ in { "net.ipv6.conf.all.forwarding" = "1"; "net.ipv6.conf.en-virginmedia.accept_ra" = "2"; "net.ipv6.conf.en-ee.accept_ra" = "2"; + "net.ipv6.conf.en-gnet.accept_ra" = "2"; }; networking.nat = { enable = true; @@ -198,19 +217,25 @@ in { # NAT packets going over EE plain. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE + # NAT packets going over GNetwork plain. + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE + # SNAT packets we're sending over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254 iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254 # eduroam # > mark incoming eduroam packets iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2 - # > NAT packets going over EE/VM. + # > NAT packets going out directly. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-virginmedia -j MASQUERADE iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE # > NAT packets going over tunnels. iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.253 iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253 + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253 ''; }; services.dhcpd4 = { @@ -324,6 +349,21 @@ in { wg set wg-tuvok-ee fwmark 0xdead ''; }; + interfaces.wg-tuvok-gnet = ifBase // { + ips = [ + "2a09:a442::3:1/112" + "92.118.30.4/31" + ]; + listenPort = 51822; + privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; + peers = [(peerBase // { + endpoint = "92.118.28.252:51822"; + publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; + })]; + postSetup = '' + wg set wg-tuvok-gnet fwmark 0xcafe + ''; + }; }; services.unifi = { @@ -360,6 +400,21 @@ in { 53 # DNS ]; }; + interfaces.en-virginmedia = { + allowedUDPPorts = [ + 51820 + ]; + }; + interfaces.en-ee = { + allowedUDPPorts = [ + 51821 + ]; + }; + interfaces.en-gnet = { + allowedUDPPorts = [ + 51822 + ]; + }; interfaces.wg-tuvok-ee = { allowedUDPPorts = [ 3784 # BFD @@ -370,6 +425,11 @@ in { 3784 # BFD ]; }; + interfaces.wg-tuvok-gnet = { + allowedUDPPorts = [ + 3784 # BFD + ]; + }; extraCommands = '' ip46tables -F FORWARD @@ -381,6 +441,7 @@ in { ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-vm -j ACCEPT + ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT ''; }; @@ -521,6 +582,10 @@ in { # EE preference = 10; }; + route 0.0.0.0/0 via 92.118.30.5 bfd { + # GNetwork + preference = 200; + }; }; protocol static export6 { ipv6 {}; @@ -534,6 +599,11 @@ in { preference = 10; krt_prefsrc = 2a09:a443::1; }; + route ::/0 via 2a09:a442::3:2 bfd { + # GNetwork + preference = 200; + krt_prefsrc = 2a09:a443::1; + }; # Covering route... route 2a09:a443::/64 via "en-general"; @@ -554,6 +624,8 @@ in { neighbor 2a09:a442::1:2; neighbor 92.118.30.3; neighbor 2a09:a442::2:2; + neighbor 92.118.30.5; + neighbor 2a09:a442::3:2; }; ''; };