swann: migrate fully to using networkd
networkd appears to have gotten very aggressive about clearing routing rules it didn't insert itself
This commit is contained in:
parent
9099ee2a45
commit
5283ee4fee
4 changed files with 276 additions and 139 deletions
|
|
@ -99,77 +99,7 @@ in {
|
|||
};
|
||||
};
|
||||
|
||||
localCommands = let
|
||||
claimedPriorities = { min = 10000; max = 10100; };
|
||||
rules = [
|
||||
# Route traffic to EE via WG... via EE.
|
||||
{ priority = 10000; both = "fwmark 0xdead table 201"; }
|
||||
# Route traffic to VM via WG... via VM DHCP in table 202.
|
||||
{ priority = 10001; both = "fwmark 0xbeef table 202"; }
|
||||
# Route traffic to GNetwork via WG... via DHCP in table 203.
|
||||
{ priority = 10002; both = "fwmark 0xcafe table 203"; }
|
||||
|
||||
# Make ping work over the tunnels.
|
||||
{ priority = 10010; v4 = "from 92.118.30.0 table 151"; v6 = "from 2a09:a441::1:1 table 151"; }
|
||||
{ priority = 10011; v4 = "from 92.118.30.2 table 152"; v6 = "from 2a09:a441::2:1 table 152"; }
|
||||
{ priority = 10012; v4 = "from 92.118.30.4 table 153"; v6 = "from 2a09:a441::3:1 table 153"; }
|
||||
|
||||
# Now some subset of RFC1918 via main table too.
|
||||
{ priority = 10020; v4 = "to 192.168.0.0/16 table main"; }
|
||||
{ priority = 10021; v4 = "to 10.0.0.0/8 table main"; }
|
||||
{ priority = 10022; v4 = "to 172.16.0.0/12 table main"; }
|
||||
|
||||
# And the linknets...
|
||||
{ priority = 10023; v4 = "to 92.118.30.0/24 table main"; }
|
||||
{ priority = 10024; v6 = "to 2a09:a441::1:0/112 table main"; }
|
||||
{ priority = 10025; v6 = "to 2a09:a441::2:0/112 table main"; }
|
||||
{ priority = 10026; v6 = "to 2a09:a441::3:0/112 table main"; }
|
||||
|
||||
# add-on.ee.co.uk goes via EE.
|
||||
{ priority = 10031; v4 = "to 82.192.97.153/32 table 201"; }
|
||||
|
||||
# Anything originating from 192.168.200.0/24 should go via EE too.
|
||||
{ priority = 10032; v4 = "from 192.168.200.0/24 table 201"; }
|
||||
|
||||
# Everything else over WG.
|
||||
{ priority = 10080; both = "table 150"; }
|
||||
|
||||
# Fallbacks via GNetwork, VM, EE
|
||||
# Sometimes this seems to be required to get things moving, for some super unclear reason.
|
||||
{ priority = 10090; both = "table 203"; }
|
||||
{ priority = 10091; both = "table 202"; }
|
||||
{ priority = 10092; both = "table 201"; }
|
||||
];
|
||||
clearRules = map (x: ''
|
||||
ip -4 rule del priority ${toString x} >/dev/null 2>&1 || true
|
||||
ip -6 rule del priority ${toString x} >/dev/null 2>&1 || true
|
||||
'') (lib.range claimedPriorities.min (claimedPriorities.max - 1));
|
||||
ruleToLine = { priority, v4 ? "", v6 ? "", both ? "" }:
|
||||
assert (both == "" || (v4 == "" && v6 == ""));
|
||||
assert priority >= claimedPriorities.min;
|
||||
assert priority < claimedPriorities.max;
|
||||
let
|
||||
rv4 = if v4 != "" then v4 else both;
|
||||
rv6 = if v6 != "" then v6 else both;
|
||||
in ''
|
||||
${if rv4 != "" then "ip -4 rule add ${rv4} priority ${toString priority}" else ""}
|
||||
${if rv6 != "" then "ip -6 rule add ${rv6} priority ${toString priority}" else ""}
|
||||
'';
|
||||
addRules = map ruleToLine rules;
|
||||
in ''
|
||||
# Fix Tailscale, by adding routing rules just before the one they add at prio 5200.
|
||||
ip -4 rule del priority 5196 || true
|
||||
ip -4 rule del priority 5197 || true
|
||||
ip -4 rule del priority 5198 || true
|
||||
ip -4 rule del priority 5199 || true
|
||||
ip -4 rule add from all fwmark 0x80000 lookup 150 priority 5196
|
||||
ip -4 rule add from all fwmark 0x80000 lookup 151 priority 5197
|
||||
ip -4 rule add from all fwmark 0x80000 lookup 152 priority 5198
|
||||
ip -4 rule add from all fwmark 0x80000 lookup 153 priority 5199
|
||||
|
||||
${lib.concatStringsSep "\n" clearRules}
|
||||
${lib.concatStringsSep "\n" addRules}
|
||||
|
||||
localCommands = ''
|
||||
ip -4 route flush table 151 >/dev/null 2>&1 || true
|
||||
ip -4 route add 92.118.30.0/31 dev wg-tuvok-vm table 151
|
||||
ip -4 route add default via 92.118.30.1 dev wg-tuvok-vm table 151
|
||||
|
|
@ -192,13 +122,245 @@ in {
|
|||
ip -6 route add default via 2a09:a442::3:2 dev wg-tuvok-gnet table 153
|
||||
'';
|
||||
};
|
||||
systemd.network = {
|
||||
systemd.network = let
|
||||
hexToInt = h: (builtins.fromTOML "h = ${h}").h;
|
||||
physicalNetwork = rtID: wireguardFwmark: extraRules: {
|
||||
dhcpV4Config.RouteTable = rtID;
|
||||
routingPolicyRules = [{
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "both";
|
||||
FirewallMark = hexToInt wireguardFwmark;
|
||||
Priority = 10000;
|
||||
Table = rtID;
|
||||
};
|
||||
}] ++ extraRules;
|
||||
};
|
||||
wireguardNetwork = { linkName, relativePriority, rtID, v4Linknet, v6Linknet }: {
|
||||
matchConfig.Name = linkName;
|
||||
|
||||
routes = let
|
||||
replaceV4Octet = v4: fn: let
|
||||
pieces = builtins.match ''(([0-9]+\.){3})([0-9]+)'' v4;
|
||||
in
|
||||
"${builtins.elemAt pieces 0}${toString (fn (lib.toInt (builtins.elemAt pieces 2)))}";
|
||||
replaceV6Octet = v6: fn: let
|
||||
pieces = builtins.match ''^(([0-9a-f]+:+)+)([0-9a-f]*)$'' v6;
|
||||
in
|
||||
"${builtins.elemAt pieces 0}${lib.toHexString (fn (hexToInt "0x${builtins.elemAt pieces 2}"))}";
|
||||
in [
|
||||
{
|
||||
routeConfig = {
|
||||
Destination = "${v4Linknet}/31";
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
{
|
||||
routeConfig = {
|
||||
Gateway = replaceV4Octet v4Linknet (n: n + 1);
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
routeConfig = {
|
||||
Destination = "${replaceV6Octet v6Linknet (n: n - 1)}/112";
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
{
|
||||
routeConfig = {
|
||||
Gateway = replaceV6Octet v6Linknet (n: n + 1);
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
networkConfig = {
|
||||
Address = [
|
||||
"${v4Linknet}/31"
|
||||
"${v6Linknet}/112"
|
||||
];
|
||||
};
|
||||
|
||||
routingPolicyRules = [
|
||||
(tailscaleRule (relativePriority + 5000) rtID)
|
||||
|
||||
# Allow picking destination by source IP.
|
||||
{
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "ipv4";
|
||||
From = v4Linknet;
|
||||
Priority = 10010;
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
{
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "ipv6";
|
||||
From = v6Linknet;
|
||||
Priority = 10010;
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
# Catch-all mop-up rule at the end.
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "both";
|
||||
Priority = relativePriority + 10090;
|
||||
Table = rtID;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
tailscaleRule = priority: table: {
|
||||
# Route Tailscale (fwmark 0x80000) via Wireguard first.
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "both";
|
||||
FirewallMark = hexToInt "0x80000";
|
||||
Priority = priority;
|
||||
Table = table;
|
||||
};
|
||||
};
|
||||
in {
|
||||
enable = true;
|
||||
networks."40-en-ee".dhcpV4Config.RouteTable = 201;
|
||||
networks."40-en-ee".linkConfig.RequiredForOnline = "no";
|
||||
networks."40-en-virginmedia".dhcpV4Config.RouteTable = 202;
|
||||
networks."40-en-virginmedia".linkConfig.RequiredForOnline = "no";
|
||||
networks."40-en-gnet".dhcpV4Config.RouteTable = 203;
|
||||
networks."50-wg-tuvok-vm" = wireguardNetwork {
|
||||
linkName = "wg-tuvok-vm";
|
||||
relativePriority = 2;
|
||||
rtID = 151;
|
||||
v4Linknet = "92.118.30.0";
|
||||
v6Linknet = "2a09:a442::1:1";
|
||||
};
|
||||
networks."50-wg-tuvok-ee" = wireguardNetwork {
|
||||
linkName = "wg-tuvok-ee";
|
||||
relativePriority = 3;
|
||||
rtID = 152;
|
||||
v4Linknet = "92.118.30.2";
|
||||
v6Linknet = "2a09:a442::2:1";
|
||||
};
|
||||
networks."50-wg-tuvok-gnet" = wireguardNetwork {
|
||||
linkName = "wg-tuvok-gnet";
|
||||
relativePriority = 1;
|
||||
rtID = 153;
|
||||
v4Linknet = "92.118.30.4";
|
||||
v6Linknet = "2a09:a442::3:1";
|
||||
};
|
||||
networks."40-lo" = {
|
||||
routingPolicyRules = let
|
||||
viaMain = priority: to: {
|
||||
routingPolicyRuleConfig = {
|
||||
To = to;
|
||||
Table = "main";
|
||||
Priority = priority;
|
||||
};
|
||||
};
|
||||
blackhole = fwmark: {
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "both";
|
||||
FirewallMark = hexToInt fwmark;
|
||||
Priority = 10001;
|
||||
Type = "unreachable";
|
||||
};
|
||||
};
|
||||
in [
|
||||
(tailscaleRule 5000 150)
|
||||
|
||||
# Blackhole connections that should be routed over individual interfaces.
|
||||
(blackhole "0xdead")
|
||||
(blackhole "0xbeef")
|
||||
(blackhole "0xcafe")
|
||||
|
||||
# RFC 1918 via main table.
|
||||
(viaMain 10020 "192.168.0.0/16")
|
||||
(viaMain 10021 "10.0.0.0/8")
|
||||
(viaMain 10022 "172.16.0.0/12")
|
||||
# and the linknets.
|
||||
(viaMain 10023 "92.118.30.0/24")
|
||||
(viaMain 10024 "2a09:a442::1:0/112")
|
||||
(viaMain 10025 "2a09:a442::2:0/112")
|
||||
(viaMain 10026 "2a09:a442::3:0/112")
|
||||
|
||||
{
|
||||
# Catch-all "go via WG"
|
||||
routingPolicyRuleConfig = {
|
||||
Family = "both";
|
||||
Priority = 10080;
|
||||
Table = 150;
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
networks."40-en-ee" = (physicalNetwork 201 "0xdead" [{
|
||||
routingPolicyRuleConfig = {
|
||||
# add-on.ee.co.uk goes via EE.
|
||||
To = "82.192.97.153/32";
|
||||
Table = 201;
|
||||
Priority = 10031;
|
||||
};
|
||||
} {
|
||||
routingPolicyRuleConfig = {
|
||||
# as does anything from 192.168.200.0/24.
|
||||
From = "192.168.200.0/24";
|
||||
Table = 201;
|
||||
Priority = 10031;
|
||||
};
|
||||
}]) // {
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
networks."40-en-virginmedia" = (physicalNetwork 202 "0xbeef" []) // {
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
networks."40-en-gnet" = (physicalNetwork 203 "0xcafe" []);
|
||||
|
||||
netdevs = let
|
||||
wireguard = { name, listenPort, privateKey, endpoint, publicKey, fwmark }: {
|
||||
netdevConfig = {
|
||||
Name = name;
|
||||
Kind = "wireguard";
|
||||
Description = "WireGuard tunnel ${name}";
|
||||
};
|
||||
wireguardConfig = {
|
||||
ListenPort = listenPort;
|
||||
PrivateKeyFile = pkgs.writeText "${name}" privateKey;
|
||||
# TODO: PrivateKeyFile
|
||||
FirewallMark = hexToInt fwmark;
|
||||
RouteTable = "off";
|
||||
};
|
||||
wireguardPeers = [{
|
||||
wireguardPeerConfig = {
|
||||
Endpoint = endpoint;
|
||||
PublicKey = publicKey;
|
||||
AllowedIPs = [
|
||||
"0.0.0.0/0"
|
||||
"::/0"
|
||||
];
|
||||
};
|
||||
}];
|
||||
};
|
||||
tuvokWireguard = args: wireguard (args // {
|
||||
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
||||
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
||||
});
|
||||
in {
|
||||
"40-wg-tuvok-vm" = tuvokWireguard {
|
||||
name = "wg-tuvok-vm";
|
||||
listenPort = 51820;
|
||||
endpoint = "92.118.28.252:51820";
|
||||
fwmark = "0xbeef";
|
||||
};
|
||||
"40-wg-tuvok-ee" = tuvokWireguard {
|
||||
name = "wg-tuvok-ee";
|
||||
listenPort = 51821;
|
||||
endpoint = "[2a09:a441::f00f]:51821";
|
||||
fwmark = "0xdead";
|
||||
};
|
||||
"40-wg-tuvok-gnet" = tuvokWireguard {
|
||||
name = "wg-tuvok-gnet";
|
||||
listenPort = 51822;
|
||||
endpoint = "92.118.28.252:51822";
|
||||
fwmark = "0xcafe";
|
||||
};
|
||||
};
|
||||
};
|
||||
my.ip.tailscale = "100.102.224.95";
|
||||
services.udev.extraRules = ''
|
||||
|
|
@ -316,64 +478,9 @@ in {
|
|||
}
|
||||
];
|
||||
};
|
||||
networking.wireguard = let
|
||||
ifBase = {
|
||||
listenPort = null;
|
||||
allowedIPsAsRoutes = false;
|
||||
};
|
||||
peerBase = {
|
||||
allowedIPs = [
|
||||
"0.0.0.0/0"
|
||||
"::/0"
|
||||
];
|
||||
};
|
||||
in {
|
||||
enable = true;
|
||||
interfaces.wg-tuvok-vm = ifBase // {
|
||||
ips = [
|
||||
"2a09:a442::1:1/112"
|
||||
"92.118.30.0/31"
|
||||
];
|
||||
listenPort = 51820;
|
||||
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
||||
peers = [(peerBase // {
|
||||
endpoint = "92.118.28.252:51820";
|
||||
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
||||
})];
|
||||
postSetup = ''
|
||||
wg set wg-tuvok-vm fwmark 0xbeef
|
||||
'';
|
||||
};
|
||||
interfaces.wg-tuvok-ee = ifBase // {
|
||||
ips = [
|
||||
"2a09:a442::2:1/112"
|
||||
"92.118.30.2/31"
|
||||
];
|
||||
listenPort = 51821;
|
||||
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
||||
peers = [(peerBase // {
|
||||
endpoint = "[2a09:a441::f00f]:51821";
|
||||
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
||||
})];
|
||||
postSetup = ''
|
||||
wg set wg-tuvok-ee fwmark 0xdead
|
||||
'';
|
||||
};
|
||||
interfaces.wg-tuvok-gnet = ifBase // {
|
||||
ips = [
|
||||
"2a09:a442::3:1/112"
|
||||
"92.118.30.4/31"
|
||||
];
|
||||
listenPort = 51822;
|
||||
privateKey = secrets.wireguard.tuvok-swann.swann.privateKey;
|
||||
peers = [(peerBase // {
|
||||
endpoint = "92.118.28.252:51822";
|
||||
publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey;
|
||||
})];
|
||||
postSetup = ''
|
||||
wg set wg-tuvok-gnet fwmark 0xcafe
|
||||
'';
|
||||
};
|
||||
systemd.services.dhcpd4 = {
|
||||
wants = [ "systemd-networkd-wait-online.service" ];
|
||||
after = [ "systemd-networkd-wait-online.service" ];
|
||||
};
|
||||
|
||||
services.unifi = {
|
||||
|
|
@ -488,6 +595,10 @@ in {
|
|||
}
|
||||
'';
|
||||
};
|
||||
systemd.services.coredns = {
|
||||
wants = [ "systemd-networkd-wait-online.service" ];
|
||||
after = [ "systemd-networkd-wait-online.service" ];
|
||||
};
|
||||
my.prometheus.additionalExporterPorts.coredns = 9153;
|
||||
networking.resolvconf.extraConfig = ''
|
||||
name_servers='127.0.0.253'
|
||||
|
|
@ -667,6 +778,10 @@ in {
|
|||
}
|
||||
'';
|
||||
};
|
||||
systemd.services.dhcpd6 = {
|
||||
wants = [ "systemd-networkd-wait-online.service" ];
|
||||
after = [ "systemd-networkd-wait-online.service" ];
|
||||
};
|
||||
|
||||
systemd.services.prometheus-bird-exporter.serviceConfig.ExecStart = lib.mkForce ''
|
||||
${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
|
||||
|
|
@ -700,9 +815,5 @@ in {
|
|||
};
|
||||
};
|
||||
|
||||
systemd.services.systemd-networkd-wait-online.serviceConfig = {
|
||||
ExecStart = [ "" "${config.systemd.package}/lib/systemd/systemd-networkd-wait-online --any" ];
|
||||
};
|
||||
|
||||
system.stateVersion = "21.03";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -281,6 +281,8 @@ let
|
|||
"PrivateKeyFile"
|
||||
"ListenPort"
|
||||
"FirewallMark"
|
||||
"RouteTable"
|
||||
"RouteMetric"
|
||||
])
|
||||
(assertInt "FirewallMark")
|
||||
(assertRange "FirewallMark" 1 4294967295)
|
||||
|
|
@ -296,6 +298,8 @@ let
|
|||
"AllowedIPs"
|
||||
"Endpoint"
|
||||
"PersistentKeepalive"
|
||||
"RouteTable"
|
||||
"RouteMetric"
|
||||
])
|
||||
(assertInt "PersistentKeepalive")
|
||||
(assertRange "PersistentKeepalive" 0 65535)
|
||||
|
|
|
|||
21
third_party/nixpkgs/patches/networkd-support-more-wg-options.patch
vendored
Normal file
21
third_party/nixpkgs/patches/networkd-support-more-wg-options.patch
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix
|
||||
--- a/nixos/modules/system/boot/networkd.nix
|
||||
+++ b/nixos/modules/system/boot/networkd.nix
|
||||
@@ -281,6 +281,8 @@ let
|
||||
"PrivateKeyFile"
|
||||
"ListenPort"
|
||||
"FirewallMark"
|
||||
+ "RouteTable"
|
||||
+ "RouteMetric"
|
||||
])
|
||||
(assertInt "FirewallMark")
|
||||
(assertRange "FirewallMark" 1 4294967295)
|
||||
@@ -296,6 +298,8 @@ let
|
||||
"AllowedIPs"
|
||||
"Endpoint"
|
||||
"PersistentKeepalive"
|
||||
+ "RouteTable"
|
||||
+ "RouteMetric"
|
||||
])
|
||||
(assertInt "PersistentKeepalive")
|
||||
(assertRange "PersistentKeepalive" 0 65535)
|
||||
1
third_party/nixpkgs/patches/series
vendored
1
third_party/nixpkgs/patches/series
vendored
|
|
@ -1,3 +1,4 @@
|
|||
nvidia-sideband-socket.patch
|
||||
pr163673.patch
|
||||
pr163678.patch
|
||||
networkd-support-more-wg-options.patch
|
||||
|
|
|
|||
Loading…
Reference in a new issue