ops/nixos: set up gnetwork link

ops/nixos/blade-tuvok/default.nix

@@ -21,7 +21,7 @@ in {
     firewall.allowedTCPPorts = [ 80 443 ];
     firewall.allowedUDPPorts = [
       # Wireguard
-      51820 51821
+      51820 51821 51822
     ];
     localCommands = ''
       # Check if we already have our little minicontainer setup
@@ -116,12 +116,13 @@ in {
       "0.0.0.0/0"
       "::/0"
     ];
+    swannPeerBase = peerBase // {
+      endpoint = null;  # dynamic
+      publicKey = secrets.wireguard.tuvok-swann.swann.publicKey;
+    };
     swannBase = ifBase // {
       privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey;
-      peers = [(peerBase // {
+      peers = [swannPeerBase];
-        endpoint = null;  # dynamic
-        publicKey = secrets.wireguard.tuvok-swann.swann.publicKey;
-      })];
     };
   in {
     enable = true;
@@ -139,6 +140,16 @@ in {
       ];
       listenPort = 51821;
     };
+    interfaces.wg-swann-gnet = swannBase // {
+      ips = [
+        "2a09:a442::3:2/112"
+        "92.118.30.5/31"
+      ];
+      listenPort = 51822;
+      peers = [(swannPeerBase // {
+        endpoint = "185.250.189.204:51822";
+      })];
+    };
   };
   environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile {
     name = "bird-wg-endpoint.conf";
@@ -170,6 +181,10 @@ in {
           # EE
           preference = 10;
         };
+        route 92.118.30.0/24 via 92.118.30.4 bfd {
+          # GNetwork
+          preference = 200;
+        };
       };
       protocol static export6 {
         ipv6 {};
@@ -181,6 +196,10 @@ in {
           # EE
           preference = 10;
         };
+        route 2a09:a443::/32 via 2a09:a442::3:1 bfd {
+          # GNetwork
+          preference = 200;
+        };
       };
 
       protocol bfd {
@@ -194,6 +213,8 @@ in {
         neighbor 2a09:a442::1:1;
         neighbor 92.118.30.2;
         neighbor 2a09:a442::2:1;
+        neighbor 92.118.30.4;
+        neighbor 2a09:a442::3:1;
       };
     '';
     checkPhase = ''

ops/nixos/lib/coredns/zones/db.2.4.4.a.9.0.a.2.ip6.arpa

@@ -3,7 +3,7 @@
 ; SPDX-License-Identifier: Apache-2.0
 
 ;              MNAME                        RNAME                  SERIAL REFRESH RETRY EXPIRE TTL
-@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 2      600     450   3600   300
+@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 3      600     450   3600   300
 
 $INCLUDE tmpl.ns
 
@@ -11,3 +11,5 @@ $INCLUDE tmpl.ns
 2.0.0.0.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net.
 1.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net.
 2.0.0.0.2.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net.
+1.0.0.0.3.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR mldn.gnet-tuvok.mldn-rd.as205479.net.
+2.0.0.0.3.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0 3600 IN PTR tuvok.gnet-tuvok.mldn-rd.as205479.net.

ops/nixos/lib/coredns/zones/db.30.118.92.in-addr.arpa

@@ -3,7 +3,7 @@
 ; SPDX-License-Identifier: Apache-2.0
 
 ;              MNAME                        RNAME                  SERIAL REFRESH RETRY EXPIRE TTL
-@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 3      600     450   3600   300
+@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 4      600     450   3600   300
 
 $INCLUDE tmpl.ns
 
@@ -11,8 +11,8 @@ $INCLUDE tmpl.ns
 1 600 IN PTR tuvok.vm-tuvok.mldn-rd.as205479.net.
 2 600 IN PTR mldn.ee-tuvok.mldn-rd.as205479.net.
 3 600 IN PTR tuvok.ee-tuvok.mldn-rd.as205479.net.
-4 600 IN PTR 92-118-30-4.ptr.as205479.net.
+4 600 IN PTR mldn.gnet-tuvok.mldn-rd.as205479.net.
-5 600 IN PTR 92-118-30-5.ptr.as205479.net.
+5 600 IN PTR tuvok.gnet-tuvok.mldn-rd.as205479.net.
 6 600 IN PTR 92-118-30-6.ptr.as205479.net.
 7 600 IN PTR 92-118-30-7.ptr.as205479.net.
 8 600 IN PTR 92-118-30-8.ptr.as205479.net.

ops/nixos/lib/coredns/zones/db.as205479.net

@@ -3,7 +3,7 @@
 ; SPDX-License-Identifier: Apache-2.0
 
 ;              MNAME                        RNAME                  SERIAL REFRESH RETRY EXPIRE TTL
-@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 43     600     450   3600   300
+@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 44     600     450   3600   300
 
 ; NB: this are also glue records in Google Domains.
 $INCLUDE tmpl.ns
@@ -105,6 +105,10 @@ mldn.ee-tuvok.mldn-rd        3600 IN A    92.118.30.2
 mldn.ee-tuvok.mldn-rd        3600 IN AAAA 2a09:a442::2:1
 tuvok.ee-tuvok.mldn-rd       3600 IN A    92.118.30.3
 tuvok.ee-tuvok.mldn-rd       3600 IN AAAA 2a09:a442::2:2
+mldn.gnet-tuvok.mldn-rd      3600 IN A    92.118.30.4
+mldn.gnet-tuvok.mldn-rd      3600 IN AAAA 2a09:a442::3:1
+tuvok.gnet-tuvok.mldn-rd     3600 IN A    92.118.30.5
+tuvok.gnet-tuvok.mldn-rd     3600 IN AAAA 2a09:a442::3:2
 mldn-rd                      3600 IN A    92.118.30.254
 mldn-rd                      3600 IN AAAA 2a09:a443::1
 eduroam.mldn-rd              3600 IN A    92.118.30.253

ops/nixos/swann/default.nix

@@ -40,13 +40,18 @@ in {
   networking = {
     # Routing tables:
     # bgp (150) -- contains default routes over WG tunnels
-    # ee (201) -- table contains a static default route via EE
+    # wg-vm (151) -- contains default routes over WG tunnels
-    # main (254) -- DHCP routes (aka VM)
+    # wg-ee (152) -- contains default routes over WG tunnels
-    # Conventional lookup order is bgp, main. ee has routing rules.
+    # wg-gnet (153) -- contains default routes over WG tunnels
+    # ee (201) -- table contains a default route via EE
+    # vm (202) -- table contains a default route via VM
+    # gnet (203) -- table contains a default route via gnetwork
+    # main (254) -- basically empty
 
     hostName = "swann"; # Define your hostname.
     domain = "int.as205479.net";
     nameservers = ["8.8.8.8" "8.8.4.4"];
+    useNetworkd = true;
     interfaces = {
       lo = {
         ipv4.addresses = [
@@ -59,16 +64,13 @@ in {
         useDHCP = true;
         macAddress = "e4:3a:6e:16:07:61";
       };
+      en-gnet = {
+        useDHCP = true;
+        # Additional options configured in networkd.
+      };
       en-ee = {
-        ipv4.addresses = [
+        useDHCP = true;
-          { address = "192.168.200.2"; prefixLength = 24; }
+        # Additional options configured in networkd.
-        ];
-        ipv4.routes = [{
-          via = "192.168.200.1";
-          address = "0.0.0.0";
-          prefixLength = 0;
-          options.table = "201";
-        }];
       };
       en-general = {
         ipv4.addresses = [
@@ -97,24 +99,20 @@ in {
       };
     };
 
-    dhcpcd.extraConfig = ''
-      interface en-virginmedia
-      metric 100
-
-      interface en-ee
-      metric 250
-    '';
     localCommands = let
       claimedPriorities = { min = 10000; max = 10100; };
       rules = [
         # Route traffic to EE via WG... via EE.
         { priority = 10000; both = "fwmark 0xdead table 201"; }
-        # Route traffic to VM via WG... via VM DHCP in main table.
+        # Route traffic to VM via WG... via VM DHCP in table 202.
-        { priority = 10001; both = "fwmark 0xbeef table main"; }
+        { priority = 10001; both = "fwmark 0xbeef table 202"; }
+        # Route traffic to GNetwork via WG... via DHCP in table 203.
+        { priority = 10002; both = "fwmark 0xcafe table 203"; }
 
         # Make ping work over the tunnels.
         { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
         { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
+        { priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; }
 
         # Now some subset of RFC1918 via main table too.
         { priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
@@ -125,9 +123,7 @@ in {
         { priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
         { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
         { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
-
+        { priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; }
-        # And the Google VIP I'm (ab)using for Stadia (see CoreDNS below).
-        { priority = 10030; v4 = "to 216.239.38.120/32 table main"; }
 
         # add-on.ee.co.uk goes via EE.
         { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
@@ -136,7 +132,13 @@ in {
         { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
 
         # Everything else over WG.
-        { priority = 10099; both = "table 150"; }
+        { priority = 10080; both = "table 150"; }
+
+        # Fallbacks via GNetwork, VM, EE
+        # Sometimes this seems to be required to get things moving, for some super unclear reason.
+        { priority = 10090; both = "table 203"; }
+        { priority = 10091; both = "table 202"; }
+        { priority = 10092; both = "table 201"; }
       ];
       clearRules = map (x: ''
         ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
@@ -166,17 +168,33 @@ in {
       ip -6 route add default via 2a09:a442::1:2 dev wg-tuvok-vm table 151
 
       ip -4 route flush table 152 >/dev/null 2>&1 || true
-      ip -4 route add 92.118.30.2/31 dev wg-tuvok-vm table 151
+      ip -4 route add 92.118.30.2/31 dev wg-tuvok-ee table 152
       ip -4 route add default via 92.118.30.3 dev wg-tuvok-ee table 152
       ip -6 route flush table 152 >/dev/null 2>&1 || true
       ip -6 route add 2a09:a442::2:0/112 dev wg-tuvok-ee table 152
       ip -6 route add default via 2a09:a442::2:2 dev wg-tuvok-ee table 152
+
+      ip -4 route flush table 153 >/dev/null 2>&1 || true
+      ip -4 route add 92.118.30.4/31 dev wg-tuvok-gnet table 153
+      ip -4 route add default via 92.118.30.5 dev wg-tuvok-gnet table 153
+      ip -6 route flush table 153 >/dev/null 2>&1 || true
+      ip -6 route add 2a09:a442::3:0/112 dev wg-tuvok-gnet table 153
+      ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153
     '';
   };
+  systemd.network = {
+    enable = true;
+    networks."40-en-ee".dhcpV4Config.RouteTable = 201;
+    networks."40-en-ee".linkConfig.RequiredForOnline = "no";
+    networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202;
+    networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no";
+    networks."40-en-gnet".dhcpV4Config.RouteTable = 203;
+  };
   my.ip.tailscale = "100.102.224.95";
   services.udev.extraRules = ''
     ATTR{address}=="e4:3a:6e:16:07:62", NAME="en-virginmedia"
     ATTR{address}=="e4:3a:6e:16:07:63", NAME="en-ee"
+    ATTR{address}=="e4:3a:6e:16:07:64", NAME="en-gnet"
     ATTR{address}=="e4:3a:6e:16:07:67", NAME="en-general"
   '';
   boot.kernel.sysctl = {
@@ -185,6 +203,7 @@ in {
     "net.ipv6.conf.all.forwarding" = "1";
     "net.ipv6.conf.en-virginmedia.accept_ra" = "2";
     "net.ipv6.conf.en-ee.accept_ra" = "2";
+    "net.ipv6.conf.en-gnet.accept_ra" = "2";
   };
   networking.nat = {
     enable = true;
@@ -198,19 +217,25 @@ in {
       # NAT packets going over EE plain.
       iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-ee -j MASQUERADE
 
+      # NAT packets going over GNetwork plain.
+      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE
+
       # SNAT packets we're sending over tunnels.
       iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.254
       iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254
+      iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254
 
       # eduroam
       # > mark incoming eduroam packets
       iptables -w -t nat -A nixos-nat-pre -i vl-eduroam -j MARK --set-mark 2
-      # > NAT packets going over EE/VM.
+      # > NAT packets going out directly.
       iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-virginmedia -j MASQUERADE
       iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE
+      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE
       # > NAT packets going over tunnels.
       iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-vm -j SNAT --to-source 92.118.30.253
       iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253
+      iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253
     '';
   };
   services.dhcpd4 = {
@@ -324,6 +349,21 @@ in {
         wg set wg-tuvok-ee fwmark 0xdead
       '';
     };
+    interfaces.wg-tuvok-gnet = ifBase // {
+      ips = [
+        "2a09:a442::3:1/112"
+        "92.118.30.4/31"
+      ];
+      listenPort = 51822;
+      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
+      peers = [(peerBase // {
+        endpoint = "92.118.28.252:51822";
+        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
+      })];
+      postSetup = ''
+        wg set wg-tuvok-gnet fwmark 0xcafe
+      '';
+    };
   };
 
   services.unifi = {
@@ -360,6 +400,21 @@ in {
         53  # DNS
       ];
     };
+    interfaces.en-virginmedia = {
+      allowedUDPPorts = [
+        51820
+      ];
+    };
+    interfaces.en-ee = {
+      allowedUDPPorts = [
+        51821
+      ];
+    };
+    interfaces.en-gnet = {
+      allowedUDPPorts = [
+        51822
+      ];
+    };
     interfaces.wg-tuvok-ee = {
       allowedUDPPorts = [
         3784  # BFD
@@ -370,6 +425,11 @@ in {
         3784  # BFD
       ];
     };
+    interfaces.wg-tuvok-gnet = {
+      allowedUDPPorts = [
+        3784  # BFD
+      ];
+    };
     extraCommands = ''
       ip46tables -F FORWARD
 
@@ -381,6 +441,7 @@ in {
 
       ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT
       ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-vm -j ACCEPT
+      ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT
       ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT
     '';
   };
@@ -521,6 +582,10 @@ in {
           # EE
           preference = 10;
         };
+        route 0.0.0.0/0 via 92.118.30.5 bfd {
+          # GNetwork
+          preference = 200;
+        };
       };
       protocol static export6 {
         ipv6 {};
@@ -534,6 +599,11 @@ in {
           preference = 10;
           krt_prefsrc = 2a09:a443::1;
         };
+        route ::/0 via 2a09:a442::3:2 bfd {
+          # GNetwork
+          preference = 200;
+          krt_prefsrc = 2a09:a443::1;
+        };
 
         # Covering route...
         route 2a09:a443::/64 via "en-general";
@@ -554,6 +624,8 @@ in {
         neighbor 2a09:a442::1:2;
         neighbor 92.118.30.3;
         neighbor 2a09:a442::2:2;
+        neighbor 92.118.30.5;
+        neighbor 2a09:a442::3:2;
       };
     '';
   };