From 5283ee4fee9da4c49707d13420d8cc3405d7a1a3 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sat, 12 Mar 2022 19:38:54 +0000 Subject: [PATCH] swann: migrate fully to using networkd networkd appears to have gotten very aggressive about clearing routing rules it didn't insert itself --- ops/nixos/swann/default.nix | 389 +++++++++++------- .../nixos/modules/system/boot/networkd.nix | 4 + .../networkd-support-more-wg-options.patch | 21 + third_party/nixpkgs/patches/series | 1 + 4 files changed, 276 insertions(+), 139 deletions(-) create mode 100644 third_party/nixpkgs/patches/networkd-support-more-wg-options.patch diff --git a/ops/nixos/swann/default.nix b/ops/nixos/swann/default.nix index e8641bc472..7209c2f2c0 100644 --- a/ops/nixos/swann/default.nix +++ b/ops/nixos/swann/default.nix @@ -99,77 +99,7 @@ in { }; }; - localCommands = let - claimedPriorities = { min = 10000; max = 10100; }; - rules = [ - # Route traffic to EE via WG... via EE. - { priority = 10000; both = "fwmark 0xdead table 201"; } - # Route traffic to VM via WG... via VM DHCP in table 202. - { priority = 10001; both = "fwmark 0xbeef table 202"; } - # Route traffic to GNetwork via WG... via DHCP in table 203. - { priority = 10002; both = "fwmark 0xcafe table 203"; } - - # Make ping work over the tunnels. - { priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; } - { priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; } - { priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; } - - # Now some subset of RFC1918 via main table too. - { priority = 10020; v4 = "to 192.168.0.0/16 table main"; } - { priority = 10021; v4 = "to 10.0.0.0/8 table main"; } - { priority = 10022; v4 = "to 172.16.0.0/12 table main"; } - - # And the linknets... - { priority = 10023; v4 = "to 92.118.30.0/24 table main"; } - { priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; } - { priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; } - { priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; } - - # add-on.ee.co.uk goes via EE. - { priority = 10031; v4 = "to 82.192.97.153/32 table 201"; } - - # Anything originating from 192.168.200.0/24 should go via EE too. - { priority = 10032; v4 = "from 192.168.200.0/24 table 201"; } - - # Everything else over WG. - { priority = 10080; both = "table 150"; } - - # Fallbacks via GNetwork, VM, EE - # Sometimes this seems to be required to get things moving, for some super unclear reason. - { priority = 10090; both = "table 203"; } - { priority = 10091; both = "table 202"; } - { priority = 10092; both = "table 201"; } - ]; - clearRules = map (x: '' - ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true - ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true - '') (lib.range claimedPriorities.min (claimedPriorities.max - 1)); - ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }: - assert (both == "" || (v4 == "" && v6 == "")); - assert priority >= claimedPriorities.min; - assert priority < claimedPriorities.max; - let - rv4 = if v4 != "" then v4 else both; - rv6 = if v6 != "" then v6 else both; - in '' - ${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""} - ${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""} - ''; - addRules = map ruleToLine rules; - in '' - # Fix Tailscale, by adding routing rules just before the one they add at prio 5200. - ip -4 rule del priority 5196 || true - ip -4 rule del priority 5197 || true - ip -4 rule del priority 5198 || true - ip -4 rule del priority 5199 || true - ip -4 rule add from all fwmark 0x80000 lookup 150 priority 5196 - ip -4 rule add from all fwmark 0x80000 lookup 151 priority 5197 - ip -4 rule add from all fwmark 0x80000 lookup 152 priority 5198 - ip -4 rule add from all fwmark 0x80000 lookup 153 priority 5199 - - ${lib.concatStringsSep "\n" clearRules} - ${lib.concatStringsSep "\n" addRules} - + localCommands = '' ip -4 route flush table 151 >/dev/null 2>&1 || true ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151 ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151 @@ -192,13 +122,245 @@ in { ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153 ''; }; - systemd.network = { + systemd.network = let + hexToInt = h: (builtins.fromTOML "h = ${h}").h; + physicalNetwork = rtID: wireguardFwmark: extraRules: { + dhcpV4Config.RouteTable = rtID; + routingPolicyRules = [{ + routingPolicyRuleConfig = { + Family = "both"; + FirewallMark = hexToInt wireguardFwmark; + Priority = 10000; + Table = rtID; + }; + }] ++ extraRules; + }; + wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: { + matchConfig.Name = linkName; + + routes = let + replaceV4Octet = v4: fn: let + pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4; + in + "${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}"; + replaceV6Octet = v6: fn: let + pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6; + in + "${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}"; + in [ + { + routeConfig = { + Destination = "${v4Linknet}/31"; + Table = rtID; + }; + } + { + routeConfig = { + Gateway = replaceV4Octet v4Linknet (n: n + 1); + Table = rtID; + }; + } + + { + routeConfig = { + Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112"; + Table = rtID; + }; + } + { + routeConfig = { + Gateway = replaceV6Octet v6Linknet (n: n + 1); + Table = rtID; + }; + } + ]; + + networkConfig = { + Address = [ + "${v4Linknet}/31" + "${v6Linknet}/112" + ]; + }; + + routingPolicyRules = [ + (tailscaleRule (relativePriority + 5000) rtID) + + # Allow picking destination by source IP. + { + routingPolicyRuleConfig = { + Family = "ipv4"; + From = v4Linknet; + Priority = 10010; + Table = rtID; + }; + } + { + routingPolicyRuleConfig = { + Family = "ipv6"; + From = v6Linknet; + Priority = 10010; + Table = rtID; + }; + } + + { + # Catch-all mop-up rule at the end. + routingPolicyRuleConfig = { + Family = "both"; + Priority = relativePriority + 10090; + Table = rtID; + }; + } + ]; + }; + tailscaleRule = priority: table: { + # Route Tailscale (fwmark 0x80000) via Wireguard first. + routingPolicyRuleConfig = { + Family = "both"; + FirewallMark = hexToInt "0x80000"; + Priority = priority; + Table = table; + }; + }; + in { enable = true; - networks."40-en-ee".dhcpV4Config.RouteTable = 201; - networks."40-en-ee".linkConfig.RequiredForOnline = "no"; - networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202; - networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no"; - networks."40-en-gnet".dhcpV4Config.RouteTable = 203; + networks."50-wg-tuvok-vm" = wireguardNetwork { + linkName = "wg-tuvok-vm"; + relativePriority = 2; + rtID = 151; + v4Linknet = "92.118.30.0"; + v6Linknet = "2a09:a442::1:1"; + }; + networks."50-wg-tuvok-ee" = wireguardNetwork { + linkName = "wg-tuvok-ee"; + relativePriority = 3; + rtID = 152; + v4Linknet = "92.118.30.2"; + v6Linknet = "2a09:a442::2:1"; + }; + networks."50-wg-tuvok-gnet" = wireguardNetwork { + linkName = "wg-tuvok-gnet"; + relativePriority = 1; + rtID = 153; + v4Linknet = "92.118.30.4"; + v6Linknet = "2a09:a442::3:1"; + }; + networks."40-lo" = { + routingPolicyRules = let + viaMain = priority: to: { + routingPolicyRuleConfig = { + To = to; + Table = "main"; + Priority = priority; + }; + }; + blackhole = fwmark: { + routingPolicyRuleConfig = { + Family = "both"; + FirewallMark = hexToInt fwmark; + Priority = 10001; + Type = "unreachable"; + }; + }; + in [ + (tailscaleRule 5000 150) + + # Blackhole connections that should be routed over individual interfaces. + (blackhole "0xdead") + (blackhole "0xbeef") + (blackhole "0xcafe") + + # RFC 1918 via main table. + (viaMain 10020 "192.168.0.0/16") + (viaMain 10021 "10.0.0.0/8") + (viaMain 10022 "172.16.0.0/12") + # and the linknets. + (viaMain 10023 "92.118.30.0/24") + (viaMain 10024 "2a09:a442::1:0/112") + (viaMain 10025 "2a09:a442::2:0/112") + (viaMain 10026 "2a09:a442::3:0/112") + + { + # Catch-all "go via WG" + routingPolicyRuleConfig = { + Family = "both"; + Priority = 10080; + Table = 150; + }; + } + ]; + }; + networks."40-en-ee" = (physicalNetwork 201 "0xdead" [{ + routingPolicyRuleConfig = { + # add-on.ee.co.uk goes via EE. + To = "82.192.97.153/32"; + Table = 201; + Priority = 10031; + }; + } { + routingPolicyRuleConfig = { + # as does anything from 192.168.200.0/24. + From = "192.168.200.0/24"; + Table = 201; + Priority = 10031; + }; + }]) // { + linkConfig.RequiredForOnline = "no"; + }; + networks."40-en-virginmedia" = (physicalNetwork 202 "0xbeef" []) // { + linkConfig.RequiredForOnline = "no"; + }; + networks."40-en-gnet" = (physicalNetwork 203 "0xcafe" []); + + netdevs = let + wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: { + netdevConfig = { + Name = name; + Kind = "wireguard"; + Description = "WireGuard tunnel ${name}"; + }; + wireguardConfig = { + ListenPort = listenPort; + PrivateKeyFile = pkgs.writeText "${name}" privateKey; + # TODO: PrivateKeyFile + FirewallMark = hexToInt fwmark; + RouteTable = "off"; + }; + wireguardPeers = [{ + wireguardPeerConfig = { + Endpoint = endpoint; + PublicKey = publicKey; + AllowedIPs = [ + "0.0.0.0/0" + "::/0" + ]; + }; + }]; + }; + tuvokWireguard = args: wireguard (args // { + privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; + publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; + }); + in { + "40-wg-tuvok-vm" = tuvokWireguard { + name = "wg-tuvok-vm"; + listenPort = 51820; + endpoint = "92.118.28.252:51820"; + fwmark = "0xbeef"; + }; + "40-wg-tuvok-ee" = tuvokWireguard { + name = "wg-tuvok-ee"; + listenPort = 51821; + endpoint = "[2a09:a441::f00f]:51821"; + fwmark = "0xdead"; + }; + "40-wg-tuvok-gnet" = tuvokWireguard { + name = "wg-tuvok-gnet"; + listenPort = 51822; + endpoint = "92.118.28.252:51822"; + fwmark = "0xcafe"; + }; + }; }; my.ip.tailscale = "100.102.224.95"; services.udev.extraRules = '' @@ -316,64 +478,9 @@ in { } ]; }; - networking.wireguard = let - ifBase = { - listenPort = null; - allowedIPsAsRoutes = false; - }; - peerBase = { - allowedIPs = [ - "0.0.0.0/0" - "::/0" - ]; - }; - in { - enable = true; - interfaces.wg-tuvok-vm = ifBase // { - ips = [ - "2a09:a442::1:1/112" - "92.118.30.0/31" - ]; - listenPort = 51820; - privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; - peers = [(peerBase // { - endpoint = "92.118.28.252:51820"; - publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; - })]; - postSetup = '' - wg set wg-tuvok-vm fwmark 0xbeef - ''; - }; - interfaces.wg-tuvok-ee = ifBase // { - ips = [ - "2a09:a442::2:1/112" - "92.118.30.2/31" - ]; - listenPort = 51821; - privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; - peers = [(peerBase // { - endpoint = "[2a09:a441::f00f]:51821"; - publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; - })]; - postSetup = '' - wg set wg-tuvok-ee fwmark 0xdead - ''; - }; - interfaces.wg-tuvok-gnet = ifBase // { - ips = [ - "2a09:a442::3:1/112" - "92.118.30.4/31" - ]; - listenPort = 51822; - privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; - peers = [(peerBase // { - endpoint = "92.118.28.252:51822"; - publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; - })]; - postSetup = '' - wg set wg-tuvok-gnet fwmark 0xcafe - ''; - }; + systemd.services.dhcpd4 = { + wants = [ "systemd-networkd-wait-online.service" ]; + after = [ "systemd-networkd-wait-online.service" ]; }; services.unifi = { @@ -488,6 +595,10 @@ in { } ''; }; + systemd.services.coredns = { + wants = [ "systemd-networkd-wait-online.service" ]; + after = [ "systemd-networkd-wait-online.service" ]; + }; my.prometheus.additionalExporterPorts.coredns = 9153; networking.resolvconf.extraConfig = '' name_servers='127.0.0.253' @@ -667,6 +778,10 @@ in { } ''; }; + systemd.services.dhcpd6 = { + wants = [ "systemd-networkd-wait-online.service" ]; + after = [ "systemd-networkd-wait-online.service" ]; + }; systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce '' ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \ @@ -700,9 +815,5 @@ in { }; }; - systemd.services.systemd-networkd-wait-online.serviceConfig = { - ExecStart = [ "" "${config.systemd.package}/lib/systemd/systemd-networkd-wait-online --any" ]; - }; - system.stateVersion = "21.03"; } diff --git a/third_party/nixpkgs/nixos/modules/system/boot/networkd.nix b/third_party/nixpkgs/nixos/modules/system/boot/networkd.nix index ac1e4ef34b..4444ce3363 100644 --- a/third_party/nixpkgs/nixos/modules/system/boot/networkd.nix +++ b/third_party/nixpkgs/nixos/modules/system/boot/networkd.nix @@ -281,6 +281,8 @@ let "PrivateKeyFile" "ListenPort" "FirewallMark" + "RouteTable" + "RouteMetric" ]) (assertInt "FirewallMark") (assertRange "FirewallMark" 1 4294967295) @@ -296,6 +298,8 @@ let "AllowedIPs" "Endpoint" "PersistentKeepalive" + "RouteTable" + "RouteMetric" ]) (assertInt "PersistentKeepalive") (assertRange "PersistentKeepalive" 0 65535) diff --git a/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch b/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch new file mode 100644 index 0000000000..929699c8d3 --- /dev/null +++ b/third_party/nixpkgs/patches/networkd-support-more-wg-options.patch @@ -0,0 +1,21 @@ +diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix +--- a/nixos/modules/system/boot/networkd.nix ++++ b/nixos/modules/system/boot/networkd.nix +@@ -281,6 +281,8 @@ let + "PrivateKeyFile" + "ListenPort" + "FirewallMark" ++ "RouteTable" ++ "RouteMetric" + ]) + (assertInt "FirewallMark") + (assertRange "FirewallMark" 1 4294967295) +@@ -296,6 +298,8 @@ let + "AllowedIPs" + "Endpoint" + "PersistentKeepalive" ++ "RouteTable" ++ "RouteMetric" + ]) + (assertInt "PersistentKeepalive") + (assertRange "PersistentKeepalive" 0 65535) diff --git a/third_party/nixpkgs/patches/series b/third_party/nixpkgs/patches/series index 98374df6c6..25caabd525 100644 --- a/third_party/nixpkgs/patches/series +++ b/third_party/nixpkgs/patches/series @@ -1,3 +1,4 @@ nvidia-sideband-socket.patch pr163673.patch pr163678.patch +networkd-support-more-wg-options.patch