From 761465b393bc1c1b791221cdfe3c4938af7fdb1f Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Thu, 4 Apr 2024 22:51:27 +0000 Subject: [PATCH] swann: add peering to cofractal-ams01/rexxar --- ops/nixos/swann/default.nix | 305 ++++++++++++++++++++++++++++++------ 1 file changed, 256 insertions(+), 49 deletions(-) diff --git a/ops/nixos/swann/default.nix b/ops/nixos/swann/default.nix index 63c16bcee9..99bfdae14f 100644 --- a/ops/nixos/swann/default.nix +++ b/ops/nixos/swann/default.nix @@ -44,8 +44,6 @@ in { networking = { # Routing tables: # bgp (150) -- contains default routes over WG tunnels - # wg-ee (152) -- contains default routes over WG tunnels - # wg-gnet (153) -- contains default routes over WG tunnels # ee (201) -- table contains a default route via EE # vm (202) -- table contains a default route via VM # gnet (203) -- table contains a default route via gnetwork @@ -195,6 +193,8 @@ in { bgp = 150; wg-ee = 152; wg-gnet = 153; + wg-cf-ee = 154; + wg-cf-gnet = 155; wg-eta = 160; ee = 201; gnet = 203; @@ -202,20 +202,34 @@ in { in { enable = true; config.routeTables = routeTables; - networks."50-wg-tuvok-ee" = wireguardNetwork { - linkName = "wg-tuvok-ee"; + networks."50-wg-rexxar-ee" = wireguardNetwork { + linkName = "wg-rexxar-ee"; relativePriority = 3; rtID = routeTables.wg-ee; v4Linknet = "92.118.30.2"; v6Linknet = "2a09:a442::2:1"; }; - networks."50-wg-tuvok-gnet" = wireguardNetwork { - linkName = "wg-tuvok-gnet"; + networks."50-wg-rexxar-gnet" = wireguardNetwork { + linkName = "wg-rexxar-gnet"; relativePriority = 1; rtID = routeTables.wg-gnet; v4Linknet = "92.118.30.4"; v6Linknet = "2a09:a442::3:1"; }; + networks."50-wg-cofractal-ee" = wireguardNetwork { + linkName = "wg-cofractal-ee"; + relativePriority = 4; + rtID = routeTables.wg-cf-ee; + v4Linknet = "92.118.30.0"; + v6Linknet = "2a09:a442::1:1"; + }; + networks."50-wg-cofractal-gn" = wireguardNetwork { + linkName = "wg-cofractal-gn"; + relativePriority = 2; + rtID = routeTables.wg-cf-gnet; + v4Linknet = "92.118.30.6"; + v6Linknet = "2a09:a442::4:1"; + }; networks."50-wg-eta" = wireguardNetwork { linkName = "wg-eta"; relativePriority = 10; @@ -326,8 +340,7 @@ in { }; wireguardConfig = { ListenPort = listenPort; - PrivateKeyFile = pkgs.writeText "${name}" privateKey; - # TODO: PrivateKeyFile + PrivateKeyFile = privateKey; FirewallMark = hexToInt fwmark; RouteTable = "off"; }; @@ -342,22 +355,39 @@ in { }; }]; }; - tuvokWireguard = args: wireguard (args // { - privateKey = secrets.wireguard.tuvok-swann.swann.privateKey; - publicKey = secrets.wireguard.tuvok-swann.tuvok.publicKey; + rexxarWireguard = args: wireguard (args // { + privateKey = config.my.vault.secrets.wg-rexxar-private.path; + publicKey = "UnmWp6OI283EJfJ9PabZePq/IDsfk1ku/vilRy9TIg0="; + }); + cofractalams01Wireguard = args: wireguard (args // { + privateKey = config.my.vault.secrets.wg-cofractal-ams01-private.path; + publicKey = "go9lHRpTrtPx8oGzsCs3bG1Z9bQBYsCk7l3T+1KBi1Y="; }); in { - "40-wg-tuvok-ee" = tuvokWireguard { - name = "wg-tuvok-ee"; + "40-wg-rexxar-ee" = rexxarWireguard { + name = "wg-rexxar-ee"; listenPort = 51821; #endpoint = "92.118.28.252:51821"; - endpoint = "[2a09:a441::f00f]:51821"; + endpoint = "[2a09:a442:1000::]:51821"; fwmark = "0xdead"; }; - "40-wg-tuvok-gnet" = tuvokWireguard { - name = "wg-tuvok-gnet"; + "40-wg-rexxar-gnet" = rexxarWireguard { + name = "wg-rexxar-gnet"; listenPort = 51822; - endpoint = "92.118.28.252:51822"; + endpoint = "92.118.30.251:51822"; + fwmark = "0xcafe"; + }; + "40-wg-cofractal-ee" = cofractalams01Wireguard { + name = "wg-cofractal-ee"; + listenPort = 51823; + #endpoint = "92.118.28.252:51821"; + endpoint = "[2a09:a446:1337:ffff::10]:51821"; + fwmark = "0xdead"; + }; + "40-wg-cofractal-gn" = cofractalams01Wireguard { + name = "wg-cofractal-gn"; + listenPort = 51824; + endpoint = "199.19.152.160:51822"; fwmark = "0xcafe"; }; "50-wg-eta" = { @@ -412,23 +442,19 @@ in { }; }; }; - my.vault.secrets = { - wg-eta-public = { + my.vault.secrets = let + wireguardSecret = key: { group = "systemd-network"; template = '' {{- with secret "kv/apps/wireguard/swann" -}} - {{- .Data.data.publicKeyFromEta -}} - {{- end -}} - ''; - }; - wg-eta-private = { - group = "systemd-network"; - template = '' - {{- with secret "kv/apps/wireguard/swann" -}} - {{- .Data.data.privateKeyToEta -}} + {{- .Data.data.${key} -}} {{- end -}} ''; }; + in { + wg-eta-private = wireguardSecret "privateKeyToEta"; + wg-rexxar-private = wireguardSecret "privateKeyToRexxar"; + wg-cofractal-ams01-private = wireguardSecret "privateKeyToCofractalAms01"; }; services.mstpd.enable = true; my.ip.tailscale = "100.102.224.95"; @@ -462,8 +488,10 @@ in { iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o en-gnet -j MASQUERADE # SNAT packets we're sending over tunnels. - iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.254 - iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-rexxar-ee -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-rexxar-gnet -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-cofractal-ee -j SNAT --to-source 92.118.30.254 + iptables -w -t nat -A nixos-nat-post -m mark --mark 1 -o wg-cofractal-gn -j SNAT --to-source 92.118.30.254 # eduroam # > mark incoming eduroam packets @@ -472,8 +500,10 @@ in { iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-ee -j MASQUERADE iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o en-gnet -j MASQUERADE # > NAT packets going over tunnels. - iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-ee -j SNAT --to-source 92.118.30.253 - iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-tuvok-gnet -j SNAT --to-source 92.118.30.253 + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-rexxar-ee -j SNAT --to-source 92.118.30.253 + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-rexxar-gnet -j SNAT --to-source 92.118.30.253 + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-cofractal-ee -j SNAT --to-source 92.118.30.253 + iptables -w -t nat -A nixos-nat-post -m mark --mark 2 -o wg-cofractal-gn -j SNAT --to-source 92.118.30.253 ''; }; services.erbium = { @@ -623,12 +653,22 @@ in { 51822 ]; }; - interfaces.wg-tuvok-ee = { + interfaces.wg-rexxar-ee = { allowedUDPPorts = [ 3784 # BFD ]; }; - interfaces.wg-tuvok-gnet = { + interfaces.wg-rexxar-gnet = { + allowedUDPPorts = [ + 3784 # BFD + ]; + }; + interfaces.wg-cofractal-ee = { + allowedUDPPorts = [ + 3784 # BFD + ]; + }; + interfaces.wg-cofractal-gn = { allowedUDPPorts = [ 3784 # BFD ]; @@ -647,8 +687,10 @@ in { iptables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 ip6tables -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1360 - ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-ee -j ACCEPT - ip46tables -A FORWARD -i vl-eduroam -o wg-tuvok-gnet -j ACCEPT + ip46tables -A FORWARD -i vl-eduroam -o wg-rexxar-ee -j ACCEPT + ip46tables -A FORWARD -i vl-eduroam -o wg-rexxar-gnet -j ACCEPT + ip46tables -A FORWARD -i vl-eduroam -o wg-cofractal-ee -j ACCEPT + ip46tables -A FORWARD -i vl-eduroam -o wg-cofractal-gn -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -o wg-eta -j ACCEPT ip46tables -A FORWARD -i vl-eduroam -m state --state NEW,RELATED -j REJECT ''; @@ -774,16 +816,20 @@ in { protocol static export4 { ipv4 {}; - route 0.0.0.0/0 via 92.118.30.1 bfd { - # Virgin Media - preference = 100; - }; route 0.0.0.0/0 via 92.118.30.3 bfd { - # EE + # EE, rexxar preference = 10; }; route 0.0.0.0/0 via 92.118.30.5 bfd { - # GNetwork + # GNetwork, rexxar + preference = 200; + }; + route 0.0.0.0/0 via 92.118.30.1 bfd { + # EE, cofractal-ams01 + preference = 10; + }; + route 0.0.0.0/0 via 92.118.30.7 bfd { + # GNetwork, cofractal-ams01 preference = 200; }; @@ -793,24 +839,30 @@ in { route 92.118.30.16/28 via "br-internal"; route 92.118.30.254/32 via "lo"; route 92.118.30.253/32 via "lo"; - # route 92.118.30.0/31 via "wg-tuvok-vm"; - route 92.118.30.2/31 via "wg-tuvok-ee"; - route 92.118.30.4/31 via "wg-tuvok-gnet"; + route 92.118.30.0/31 via "wg-cofractal-ee"; + route 92.118.30.2/31 via "wg-rexxar-ee"; + route 92.118.30.4/31 via "wg-rexxar-gnet"; + route 92.118.30.6/31 via "wg-cofractal-gn"; }; protocol static export6 { ipv6 {}; route ::/0 via 2a09:a442::1:2 bfd { - # Virgin Media - preference = 100; + # EE, cofractal-ams01 + preference = 10; krt_prefsrc = 2a09:a443::1; }; route ::/0 via 2a09:a442::2:2 bfd { - # EE + # EE, rexxar preference = 10; krt_prefsrc = 2a09:a443::1; }; route ::/0 via 2a09:a442::3:2 bfd { - # GNetwork + # GNetwork, rexxar + preference = 200; + krt_prefsrc = 2a09:a443::1; + }; + route ::/0 via 2a09:a442::4:2 bfd { + # GNetwork, cofractal-ams01 preference = 200; krt_prefsrc = 2a09:a443::1; }; @@ -838,6 +890,161 @@ in { neighbor 2a09:a442::2:2; neighbor 92.118.30.5; neighbor 2a09:a442::3:2; + neighbor 92.118.30.7; + neighbor 2a09:a442::4:2; + }; + + protocol bgp rexxar_ee4 { + local 92.118.30.2 as 205479; + neighbor 92.118.30.3 as 205479; + interface "wg-rexxar-ee"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp rexxar_ee6 { + local 2a09:a442::2:1 as 205479; + neighbor 2a09:a442::2:2 as 205479; + interface "wg-rexxar-ee"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp rexxar_gnet4 { + local 92.118.30.4 as 205479; + neighbor 92.118.30.5 as 205479; + interface "wg-rexxar-gnet"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp rexxar_gnet6 { + local 2a09:a442::3:1 as 205479; + neighbor 2a09:a442::3:2 as 205479; + interface "wg-rexxar-gnet"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp cofractal_ee4 { + local 92.118.30.0 as 205479; + neighbor 92.118.30.1 as 205479; + interface "wg-cofractal-ee"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp cofractal_ee6 { + local 2a09:a442::1:1 as 205479; + neighbor 2a09:a442::1:2 as 205479; + interface "wg-cofractal-ee"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp cofractal_gnet4 { + local 92.118.30.6 as 205479; + neighbor 92.118.30.7 as 205479; + interface "wg-cofractal-gn"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; + }; + protocol bgp cofractal_gnet6 { + local 2a09:a442::4:1 as 205479; + neighbor 2a09:a442::4:2 as 205479; + interface "wg-cofractal-gn"; + direct; + bfd on; + + ipv4 { + export filter { + if source != RTS_STATIC then reject; + if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; + accept; + }; + import filter { + accept; + }; + next hop self; + }; }; protocol bgp eta4 {