swann: migrate fully to using networkd

networkd appears to have gotten very aggressive about clearing routing rules it didn't insert itself

ops/nixos/swann/default.nix

@@ -99,77 +99,7 @@ in {
       };
     };
 
-    localCommands = let
+    localCommands = ''
-      claimedPriorities = { min = 10000; max = 10100; };
-      rules = [
-        # Route traffic to EE via WG... via EE.
-        { priority = 10000; both = "fwmark 0xdead table 201"; }
-        # Route traffic to VM via WG... via VM DHCP in table 202.
-        { priority = 10001; both = "fwmark 0xbeef table 202"; }
-        # Route traffic to GNetwork via WG... via DHCP in table 203.
-        { priority = 10002; both = "fwmark 0xcafe table 203"; }
-
-        # Make ping work over the tunnels.
-        { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
-        { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
-        { priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; }
-
-        # Now some subset of RFC1918 via main table too.
-        { priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
-        { priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
-        { priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
-
-        # And the linknets...
-        { priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
-        { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
-        { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
-        { priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; }
-
-        # add-on.ee.co.uk goes via EE.
-        { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
-
-        # Anything originating from 192.168.200.0/24 should go via EE too.
-        { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
-
-        # Everything else over WG.
-        { priority = 10080; both = "table 150"; }
-
-        # Fallbacks via GNetwork, VM, EE
-        # Sometimes this seems to be required to get things moving, for some super unclear reason.
-        { priority = 10090; both = "table 203"; }
-        { priority = 10091; both = "table 202"; }
-        { priority = 10092; both = "table 201"; }
-      ];
-      clearRules = map (x: ''
-        ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
-        ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
-      '') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
-      ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
-        assert (both == "" || (v4 == "" && v6 == ""));
-        assert priority >= claimedPriorities.min;
-        assert priority < claimedPriorities.max;
-        let
-          rv4 = if v4 != "" then v4 else both;
-          rv6 = if v6 != "" then v6 else both;
-        in ''
-          ${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
-          ${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
-        '';
-      addRules = map ruleToLine rules;
-    in ''
-      # Fix Tailscale, by adding routing rules just before the one they add at prio 5200.
-      ip -4 rule del priority 5196 || true
-      ip -4 rule del priority 5197 || true
-      ip -4 rule del priority 5198 || true
-      ip -4 rule del priority 5199 || true
-      ip -4 rule add from all fwmark 0x80000 lookup 150 priority 5196
-      ip -4 rule add from all fwmark 0x80000 lookup 151 priority 5197
-      ip -4 rule add from all fwmark 0x80000 lookup 152 priority 5198
-      ip -4 rule add from all fwmark 0x80000 lookup 153 priority 5199
-
-      ${lib.concatStringsSep "\n" clearRules}
-      ${lib.concatStringsSep "\n" addRules}
-
       ip -4 route flush table 151 >/dev/null 2>&1 || true
       ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
       ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
@@ -192,13 +122,245 @@ in {
       ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153
     '';
   };
-  systemd.network = {
+  systemd.network = let
+    hexToInt = h: (builtins.fromTOML "h = ${h}").h;
+    physicalNetwork = rtID: wireguardFwmark: extraRules: {
+      dhcpV4Config.RouteTable = rtID;
+      routingPolicyRules = [{
+        routingPolicyRuleConfig = {
+          Family = "both";
+          FirewallMark = hexToInt wireguardFwmark;
+          Priority = 10000;
+          Table = rtID;
+        };
+      }] ++ extraRules;
+    };
+    wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: {
+      matchConfig.Name = linkName;
+
+      routes = let
+        replaceV4Octet = v4: fn: let
+          pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4;
+        in
+          "${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}";
+        replaceV6Octet = v6: fn: let
+          pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6;
+        in
+          "${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}";
+      in [
+        {
+          routeConfig = {
+            Destination = "${v4Linknet}/31";
+            Table = rtID;
+          };
+        }
+        {
+          routeConfig = {
+            Gateway = replaceV4Octet v4Linknet (n: n + 1);
+            Table = rtID;
+          };
+        }
+
+        {
+          routeConfig = {
+            Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112";
+            Table = rtID;
+          };
+        }
+        {
+          routeConfig = {
+            Gateway = replaceV6Octet v6Linknet (n: n + 1);
+            Table = rtID;
+          };
+        }
+      ];
+
+      networkConfig = {
+        Address = [
+          "${v4Linknet}/31"
+          "${v6Linknet}/112"
+        ];
+      };
+
+      routingPolicyRules = [
+        (tailscaleRule (relativePriority + 5000) rtID)
+
+        # Allow picking destination by source IP.
+        {
+          routingPolicyRuleConfig = {
+            Family = "ipv4";
+            From = v4Linknet;
+            Priority = 10010;
+            Table = rtID;
+          };
+        }
+        {
+          routingPolicyRuleConfig = {
+            Family = "ipv6";
+            From = v6Linknet;
+            Priority = 10010;
+            Table = rtID;
+          };
+        }
+
+        {
+          # Catch-all mop-up rule at the end.
+          routingPolicyRuleConfig = {
+            Family = "both";
+            Priority = relativePriority + 10090;
+            Table = rtID;
+          };
+        }
+      ];
+    };
+    tailscaleRule = priority: table: {
+      # Route Tailscale (fwmark 0x80000) via Wireguard first.
+      routingPolicyRuleConfig = {
+        Family = "both";
+        FirewallMark = hexToInt "0x80000";
+        Priority = priority;
+        Table = table;
+      };
+    };
+  in {
     enable = true;
-    networks."40-en-ee".dhcpV4Config.RouteTable = 201;
+    networks."50-wg-tuvok-vm" = wireguardNetwork {
-    networks."40-en-ee".linkConfig.RequiredForOnline = "no";
+      linkName = "wg-tuvok-vm";
-    networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202;
+      relativePriority = 2;
-    networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no";
+      rtID = 151;
-    networks."40-en-gnet".dhcpV4Config.RouteTable = 203;
+      v4Linknet = "92.118.30.0";
+      v6Linknet = "2a09:a442::1:1";
+    };
+    networks."50-wg-tuvok-ee" = wireguardNetwork {
+      linkName = "wg-tuvok-ee";
+      relativePriority = 3;
+      rtID = 152;
+      v4Linknet = "92.118.30.2";
+      v6Linknet = "2a09:a442::2:1";
+    };
+    networks."50-wg-tuvok-gnet" = wireguardNetwork {
+      linkName = "wg-tuvok-gnet";
+      relativePriority = 1;
+      rtID = 153;
+      v4Linknet = "92.118.30.4";
+      v6Linknet = "2a09:a442::3:1";
+    };
+    networks."40-lo" = {
+      routingPolicyRules = let
+        viaMain = priority: to: {
+          routingPolicyRuleConfig = {
+            To = to;
+            Table = "main";
+            Priority = priority;
+          };
+        };
+        blackhole = fwmark: {
+          routingPolicyRuleConfig = {
+            Family = "both";
+            FirewallMark = hexToInt fwmark;
+            Priority = 10001;
+            Type = "unreachable";
+          };
+        };
+      in [
+        (tailscaleRule 5000 150)
+
+        # Blackhole connections that should be routed over individual interfaces.
+        (blackhole "0xdead")
+        (blackhole "0xbeef")
+        (blackhole "0xcafe")
+
+        # RFC 1918 via main table.
+        (viaMain 10020 "192.168.0.0/16")
+        (viaMain 10021 "10.0.0.0/8")
+        (viaMain 10022 "172.16.0.0/12")
+        # and the linknets.
+        (viaMain 10023 "92.118.30.0/24")
+        (viaMain 10024 "2a09:a442::1:0/112")
+        (viaMain 10025 "2a09:a442::2:0/112")
+        (viaMain 10026 "2a09:a442::3:0/112")
+
+        {
+          # Catch-all "go via WG"
+          routingPolicyRuleConfig = {
+            Family = "both";
+            Priority = 10080;
+            Table = 150;
+          };
+        }
+      ];
+    };
+    networks."40-en-ee" = (physicalNetwork 201 "0xdead" [{
+      routingPolicyRuleConfig = {
+        # add-on.ee.co.uk goes via EE.
+        To = "82.192.97.153/32";
+        Table = 201;
+        Priority = 10031;
+      };
+    } {
+      routingPolicyRuleConfig = {
+        # as does anything from 192.168.200.0/24.
+        From = "192.168.200.0/24";
+        Table = 201;
+        Priority = 10031;
+      };
+    }]) // {
+      linkConfig.RequiredForOnline = "no";
+    };
+    networks."40-en-virginmedia" = (physicalNetwork 202 "0xbeef" []) // {
+      linkConfig.RequiredForOnline = "no";
+    };
+    networks."40-en-gnet" = (physicalNetwork 203 "0xcafe" []);
+
+    netdevs = let
+      wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: {
+        netdevConfig = {
+          Name = name;
+          Kind = "wireguard";
+          Description = "WireGuard tunnel ${name}";
+        };
+        wireguardConfig = {
+          ListenPort = listenPort;
+          PrivateKeyFile = pkgs.writeText "${name}" privateKey;
+          # TODO: PrivateKeyFile
+          FirewallMark = hexToInt fwmark;
+          RouteTable = "off";
+        };
+        wireguardPeers = [{
+          wireguardPeerConfig = {
+            Endpoint = endpoint;
+            PublicKey = publicKey;
+            AllowedIPs = [
+              "0.0.0.0/0"
+              "::/0"
+            ];
+          };
+        }];
+      };
+      tuvokWireguard = args: wireguard (args // {
+        privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
+        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
+      });
+    in {
+      "40-wg-tuvok-vm" = tuvokWireguard {
+        name = "wg-tuvok-vm";
+        listenPort = 51820;
+        endpoint = "92.118.28.252:51820";
+        fwmark = "0xbeef";
+      };
+      "40-wg-tuvok-ee" = tuvokWireguard {
+        name = "wg-tuvok-ee";
+        listenPort = 51821;
+        endpoint = "[2a09:a441::f00f]:51821";
+        fwmark = "0xdead";
+      };
+      "40-wg-tuvok-gnet" = tuvokWireguard {
+        name = "wg-tuvok-gnet";
+        listenPort = 51822;
+        endpoint = "92.118.28.252:51822";
+        fwmark = "0xcafe";
+      };
+    };
   };
   my.ip.tailscale = "100.102.224.95";
   services.udev.extraRules = ''
@@ -316,64 +478,9 @@ in {
       }
     ];
   };
-  networking.wireguard = let
+  systemd.services.dhcpd4 = {
-    ifBase = {
+    wants = [ "systemd-networkd-wait-online.service" ];
-      listenPort = null;
+    after = [ "systemd-networkd-wait-online.service" ];
-      allowedIPsAsRoutes = false;
-    };
-    peerBase = {
-      allowedIPs = [
-        "0.0.0.0/0"
-        "::/0"
-      ];
-    };
-  in {
-    enable = true;
-    interfaces.wg-tuvok-vm = ifBase // {
-      ips = [
-        "2a09:a442::1:1/112"
-        "92.118.30.0/31"
-      ];
-      listenPort = 51820;
-      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
-      peers = [(peerBase // {
-        endpoint = "92.118.28.252:51820";
-        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
-      })];
-      postSetup = ''
-        wg set wg-tuvok-vm fwmark 0xbeef
-      '';
-    };
-    interfaces.wg-tuvok-ee = ifBase // {
-      ips = [
-        "2a09:a442::2:1/112"
-        "92.118.30.2/31"
-      ];
-      listenPort = 51821;
-      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
-      peers = [(peerBase // {
-        endpoint = "[2a09:a441::f00f]:51821";
-        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
-      })];
-      postSetup = ''
-        wg set wg-tuvok-ee fwmark 0xdead
-      '';
-    };
-    interfaces.wg-tuvok-gnet = ifBase // {
-      ips = [
-        "2a09:a442::3:1/112"
-        "92.118.30.4/31"
-      ];
-      listenPort = 51822;
-      privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
-      peers = [(peerBase // {
-        endpoint = "92.118.28.252:51822";
-        publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
-      })];
-      postSetup = ''
-        wg set wg-tuvok-gnet fwmark 0xcafe
-      '';
-    };
   };
 
   services.unifi = {
@@ -488,6 +595,10 @@ in {
       }
     '';
   };
+  systemd.services.coredns = {
+    wants = [ "systemd-networkd-wait-online.service" ];
+    after = [ "systemd-networkd-wait-online.service" ];
+  };
   my.prometheus.additionalExporterPorts.coredns = 9153;
   networking.resolvconf.extraConfig = ''
     name_servers='127.0.0.253'
@@ -667,6 +778,10 @@ in {
       }
     '';
   };
+  systemd.services.dhcpd6 = {
+    wants = [ "systemd-networkd-wait-online.service" ];
+    after = [ "systemd-networkd-wait-online.service" ];
+  };
 
   systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
     ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
@@ -700,9 +815,5 @@ in {
     };
   };
 
-  systemd.services.systemd-networkd-wait-online.serviceConfig = {
-    ExecStart = [ "" "${config.systemd.package}/lib/systemd/systemd-networkd-wait-online --any" ];
-  };
-
   system.stateVersion = "21.03";
 }

third_party/nixpkgs/nixos/modules/system/boot/networkd.nix

@@ -281,6 +281,8 @@ let
           "PrivateKeyFile"
           "ListenPort"
           "FirewallMark"
+          "RouteTable"
+          "RouteMetric"
         ])
         (assertInt "FirewallMark")
         (assertRange "FirewallMark" 1 4294967295)
@@ -296,6 +298,8 @@ let
           "AllowedIPs"
           "Endpoint"
           "PersistentKeepalive"
+          "RouteTable"
+          "RouteMetric"
         ])
         (assertInt "PersistentKeepalive")
         (assertRange "PersistentKeepalive" 0 65535)

third_party/nixpkgs/patches/networkd-support-more-wg-options.patch

@@ -0,0 +1,21 @@
+diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix
+--- a/nixos/modules/system/boot/networkd.nix
++++ b/nixos/modules/system/boot/networkd.nix
+@@ -281,6 +281,8 @@ let
+           "PrivateKeyFile"
+           "ListenPort"
+           "FirewallMark"
++          "RouteTable"
++          "RouteMetric"
+         ])
+         (assertInt "FirewallMark")
+         (assertRange "FirewallMark" 1 4294967295)
+@@ -296,6 +298,8 @@ let
+           "AllowedIPs"
+           "Endpoint"
+           "PersistentKeepalive"
++          "RouteTable"
++          "RouteMetric"
+         ])
+         (assertInt "PersistentKeepalive")
+         (assertRange "PersistentKeepalive" 0 65535)

third_party/nixpkgs/patches/series

@@ -1,3 +1,4 @@
 nvidia-sideband-socket.patch
 pr163673.patch
 pr163678.patch
+networkd-support-more-wg-options.patch