From 52a0d29396c83a59280ecedab7ea161676d31950 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sat, 13 Jul 2024 16:57:05 +0000 Subject: [PATCH] nixos/bgp: fixup --- ops/nixos/cofractal-ams01/default.nix | 55 +++++++++++++++++++-- ops/nixos/lib/bgp.nix | 52 +++++++++++++++++++- ops/nixos/rexxar/bgp.nix | 59 ++++++++++++++++++++++ ops/nixos/rexxar/default.nix | 5 +- ops/nixos/swann/default.nix | 70 ++++++++++++++++++++++++--- 5 files changed, 228 insertions(+), 13 deletions(-) diff --git a/ops/nixos/cofractal-ams01/default.nix b/ops/nixos/cofractal-ams01/default.nix index 3d4457b699..dd17efdbb5 100644 --- a/ops/nixos/cofractal-ams01/default.nix +++ b/ops/nixos/cofractal-ams01/default.nix @@ -259,14 +259,14 @@ in matchConfig.Name = "wg-swann-ee"; address = [ "92.118.30.1/31" - "2a09:a442::1:2/64" + "2a09:a442::1:2/112" ]; }; networks."40-wg-swann-gnet" = { matchConfig.Name = "wg-swann-gnet"; address = [ "92.118.30.7/31" - "2a09:a442::4:2/64" + "2a09:a442::4:2/112" ]; }; networks."40-wg-rexxar" = { @@ -367,7 +367,16 @@ in local.asn = 205479; in { enable = true; - config = { + config = let + peer.swann = { + local.asn = 205479; + remote.bfd = true; + remote.asn = 205479; + remote.must_be_next_hop = false; + remote.export_community = 10; + remote.export_internal = true; + }; + in { local = { routerID = "199.19.152.160"; }; @@ -375,7 +384,34 @@ in export.v6 = [ "2a09:a446:1337::/48" "2a09:a442::/48" "2a09:a442:2000::/48" ]; internal.export.v4 = [ "92.118.30.252/32" ]; + internal.export.v4Extra = '' + route 92.118.30.0/31 via "wg-swann-ee"; + route 92.118.30.6/31 via "wg-swann-gnet"; + ''; internal.export.v6 = [ "2a09:a446:1337::/48" "2a09:a442:2000::/48" "2a09:a442::/48" ]; + internal.export.v6Extra = '' + route 2a09:a442::1:0/112 via "wg-swann-ee"; + route 2a09:a442::4:0/112 via "wg-swann-gnet"; + ''; + + peering.swann_ee = lib.mkMerge [peer.swann { + local.v4 = "92.118.30.1"; + local.v6 = "2a09:a442::1:2"; + remote.routers = [{ + enabled = true; + v4 = "92.118.30.0"; + v6 = "2a09:a442::1:1"; + }]; + }]; + peering.swann_gnet = lib.mkMerge [peer.swann { + local.v4 = "92.118.30.7"; + local.v6 = "2a09:a442::4:2"; + remote.routers = [{ + enabled = true; + v4 = "92.118.30.6"; + v6 = "2a09:a442::4:1"; + }]; + }]; peering.cofractal = { local = local // { @@ -408,6 +444,19 @@ in set_imported_next_hop_to = "2a09:a446:1337:ffff::10"; }; }; + + bfd = '' + interface "*" { + min rx interval 10ms; + min tx interval 50ms; + idle tx interval 1s; + multiplier 20; + }; + neighbor 92.118.30.0; + neighbor 2a09:a442::1:1; + neighbor 92.118.30.6; + neighbor 2a09:a442::4:1; + ''; }; }; diff --git a/ops/nixos/lib/bgp.nix b/ops/nixos/lib/bgp.nix index 89d7328680..12a3f4d26d 100644 --- a/ops/nixos/lib/bgp.nix +++ b/ops/nixos/lib/bgp.nix @@ -33,6 +33,7 @@ let } filter bgp_export_${ixName}4 { + ${if ix.remote.export_internal then "if ((ro, ${toString ix.local.asn}, 10) ~ bgp_ext_community) then accept;" else ""} if ! ((ro, ${toString ix.local.asn}, 1000) ~ bgp_ext_community) then reject; bgp_ext_community.delete([(ro, ${toString ix.local.asn}, *)]); accept; @@ -60,6 +61,7 @@ let } filter bgp_export_${ixName}6 { + ${if ix.remote.export_internal then "if ((ro, ${toString ix.local.asn}, 10) ~ bgp_ext_community) then accept;" else ""} if ! ((ro, ${toString ix.local.asn}, 1000) ~ bgp_ext_community) then reject; bgp_ext_community.delete([(ro, ${toString ix.local.asn}, *)]); accept; @@ -79,6 +81,7 @@ let multihopSnippet = { multihop ? null, ... }: if multihop == null then "# not multihop" else "multihop ${toString multihop};"; nexthopSnippet = { next_hop ? null, ... }: if next_hop == null then "# no next hop override" else "next hop ${toString next_hop};"; passiveSnippet = { passive, ... }: "passive ${if passive then "on" else "off"};"; + bfdSnippet = { bfd, ... }: "bfd ${if bfd then "on" else "off"};"; prefixLimitSnippet = limit: if limit == null then "# no import limit" else "import limit ${toString limit} action restart;"; generateSnippetForRouter = { ixName, ix, routerNum, router, ... }: '' ${lib.optionalString (ix.local.v4 != null) '' @@ -88,6 +91,7 @@ let ${multihopSnippet router} ${nexthopSnippet router} ${passiveSnippet ix.remote} + ${bfdSnippet ix.remote} local ${ix.local.v4} as ${toString ix.local.asn}; neighbor ${router.v4} as ${toString ix.remote.asn}; graceful restart on; @@ -106,6 +110,7 @@ let ${passwordSnippet router} ${multihopSnippet router} ${passiveSnippet ix.remote} + ${bfdSnippet ix.remote} local ${ix.local.v6} as ${toString ix.local.asn}; neighbor ${router.v6} as ${toString ix.remote.asn}; graceful restart on; @@ -221,6 +226,14 @@ in { type = bool; default = false; }; + bfd = mkOption { # lukegbgp.config.peering..remote.bfd + type = bool; + default = false; + }; + export_internal = mkOption { # lukegbgp.config.peering..remote.export_internal + type = bool; + default = false; + }; routers = mkOption { # lukegbgp.config.peering..remote.routers type = listOf (submodule { options = { @@ -301,6 +314,29 @@ in { }; }; }; + internal.local = mkOption { + default = { v4 = []; v6 = []; }; + type = submodule { + options = { + v4 = mkOption { # lukegbgp.config.internal.local.v4 + type = listOf str; + default = []; + }; + v4Extra = mkOption { #lukegbgp.config.internal.local.v4Extra + type = lines; + default = ""; + }; + v6 = mkOption { # lukegbgp.config.internal.local.v6 + type = listOf str; + default = []; + }; + v6Extra = mkOption { #lukegbgp.config.internal.local.v6Extra + type = lines; + default = ""; + }; + }; + }; + }; bfd = mkOption { # lukegbgp.config.bfd type = lines; default = ""; @@ -384,6 +420,7 @@ in { protocol static export4 { ipv4 { import filter { + bgp_ext_community.add((ro, 205479, 10)); bgp_ext_community.add((ro, 205479, 1000)); bgp_ext_community.add((ro, 205479, 2000)); bgp_ext_community.add((ro, 205479, 2001)); @@ -405,6 +442,7 @@ in { bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server + bgp_ext_community.add((ro, 205479, 7000)); # 1299 from rexxar # do not export to clouvider; they do... strange things. bgp_ext_community.add((rt, 0, 62240)); @@ -425,9 +463,15 @@ in { ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.internal.export.v4} ${config.services.lukegbgp.config.internal.export.v4Extra} }; + protocol static localinternal4 { + ipv4 { import all; }; + ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.internal.local.v4} + ${config.services.lukegbgp.config.internal.local.v4Extra} + }; protocol static export6 { ipv6 { import filter { + bgp_ext_community.add((ro, 205479, 10)); bgp_ext_community.add((ro, 205479, 1000)); # export bgp_ext_community.add((ro, 205479, 2000)); # bgp_ext_community.add((ro, 205479, 2001)); # @@ -449,6 +493,7 @@ in { bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server + bgp_ext_community.add((ro, 205479, 7000)); # 1299 from rexxar # do not export to clouvider; they do... strange things. bgp_ext_community.add((rt, 0, 62240)); @@ -462,13 +507,18 @@ in { protocol static exportinternal6 { ipv6 { import filter { - bgp_ext_community.add((ro, 205479, 10)); # internal only + bgp_ext_community.add((ro, 205479, 10)); # internal only accept; }; }; ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.internal.export.v6} ${config.services.lukegbgp.config.internal.export.v6Extra} }; + protocol static localinternal6 { + ipv6 { import all; }; + ${lib.concatMapStrings (ip: "route ${ip} blackhole;") config.services.lukegbgp.config.internal.local.v6} + ${config.services.lukegbgp.config.internal.local.v6Extra} + }; protocol bfd { ${config.services.lukegbgp.config.bfd} diff --git a/ops/nixos/rexxar/bgp.nix b/ops/nixos/rexxar/bgp.nix index 77dadb6f31..5469066617 100644 --- a/ops/nixos/rexxar/bgp.nix +++ b/ops/nixos/rexxar/bgp.nix @@ -20,7 +20,15 @@ export.v6 = [ "2a09:a442:1000::/48" "2a09:a442::/48" ]; internal.export.v4 = [ "92.118.30.251/32" ]; + internal.export.v4Extra = '' + route 92.118.30.2/31 via "wg-swann-ee"; + route 92.118.30.4/31 via "wg-swann-gnet"; + ''; internal.export.v6 = [ "2a09:a442:1000::/48" "2a09:a442::/48" ]; + internal.export.v6Extra = '' + route 2a09:a442::2:0/112 via "wg-swann-ee"; + route 2a09:a442::3:0/112 via "wg-swann-gnet"; + ''; peering = let base = [{ @@ -47,7 +55,31 @@ export_community = 4001; bgp_local_pref = 101; }; + peer.swann = { + remote.bfd = true; + remote.asn = 205479; + remote.must_be_next_hop = false; + remote.export_community = 10; + }; in { + swann_ee = lib.mkMerge (base ++ [peer.swann { + local.v4 = "92.118.30.3"; + local.v6 = "2a09:a442::2:2"; + remote.routers = [{ + enabled = true; + v4 = "92.118.30.2"; + v6 = "2a09:a442::2:1"; + }]; + }]); + swann_gnet = lib.mkMerge (base ++ [peer.swann { + local.v4 = "92.118.30.5"; + local.v6 = "2a09:a442::3:2"; + remote.routers = [{ + enabled = true; + v4 = "92.118.30.4"; + v6 = "2a09:a442::3:1"; + }]; + }]); veloxserv1 = lib.mkMerge (link.velox1 ++ [peer.velox { remote.routers = [{ enabled = true; @@ -245,7 +277,34 @@ prefix_limit.v6 = 200; }; }]); + arelion = lib.mkMerge (base ++ [{ + local = { + v4 = "62.115.150.105"; + v6 = "2001:2035:0:1f7e::2"; + }; + remote = { + asn = 1299; + export_community = 7000; + bgp_local_pref = 101; + routers = [{ + v4 = "62.115.150.104"; + v6 = "2001:2035:0:1f7e::1"; + }]; + }; + }]); }; + bfd = '' + interface "*" { + min rx interval 10ms; + min tx interval 50ms; + idle tx interval 1s; + multiplier 20; + }; + neighbor 92.118.30.4; + neighbor 2a09:a442::2:1; + neighbor 92.118.30.6; + neighbor 2a09:a442::3:1; + ''; }; }; } diff --git a/ops/nixos/rexxar/default.nix b/ops/nixos/rexxar/default.nix index 36d27b30c8..afa34174dd 100644 --- a/ops/nixos/rexxar/default.nix +++ b/ops/nixos/rexxar/default.nix @@ -91,6 +91,7 @@ 51821 51822 51823 34197 # factorio 443 + 3784 # BFD ]; allowedTCPPorts = [ 80 443 @@ -145,14 +146,14 @@ matchConfig.Name = "wg-swann-ee"; address = [ "92.118.30.3/31" - "2a09:a442::2:2/64" + "2a09:a442::2:2/112" ]; }; networks."40-wg-swann-gnet" = { matchConfig.Name = "wg-swann-gnet"; address = [ "92.118.30.5/31" - "2a09:a442::3:2/64" + "2a09:a442::3:2/112" ]; }; networks."40-wg-cofractal" = { diff --git a/ops/nixos/swann/default.nix b/ops/nixos/swann/default.nix index 8f6239f99b..9b204e93b6 100644 --- a/ops/nixos/swann/default.nix +++ b/ops/nixos/swann/default.nix @@ -406,7 +406,7 @@ in { }; }; }; - my.vault.secrets = let + my.vault.secrets = let wireguardSecret = key: { group = "systemd-network"; template = '' @@ -779,7 +779,34 @@ in { protocol device {}; protocol static export4 { - ipv4 {}; + ipv4 { + import filter { + if (ro, 205479, 1000) ~ bgp_ext_community then { + bgp_ext_community.add((ro, 205479, 2000)); # + bgp_ext_community.add((ro, 205479, 2001)); # + bgp_ext_community.add((ro, 205479, 2002)); # + bgp_ext_community.add((ro, 205479, 2003)); # + bgp_ext_community.add((ro, 205479, 3000)); # clouvider from clouvider-lon01 + bgp_ext_community.add((ro, 205479, 4000)); # frantech from frantech-nyc01/veloxserv from etheroute-lon01 + bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from rexxar + bgp_ext_community.add((ro, 205479, 4002)); # mercury from etheroute-lon01 + bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from rexxar + bgp_ext_community.add((ro, 205479, 5001)); # linx route server from rexxar + bgp_ext_community.add((ro, 205479, 5002)); # facebook from rexxar + bgp_ext_community.add((ro, 205479, 5003)); # openpeering from rexxar + bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from rexxar + bgp_ext_community.add((ro, 205479, 5005)); # he from rexxar + bgp_ext_community.add((ro, 205479, 5006)); # clouvider from rexxar + bgp_ext_community.add((ro, 205479, 5007)); # google from rexxar + bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from rexxar + bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar + bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar + bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server + bgp_ext_community.add((ro, 205479, 7000)); # 1299 from rexxar + } + accept; + }; + }; route 0.0.0.0/0 via 92.118.30.3 bfd { # EE, rexxar preference = 10; @@ -798,6 +825,7 @@ in { }; route 92.118.30.0/24 unreachable { + bgp_ext_community.add((ro, 205479, 10)); # export this - internal bgp_ext_community.add((ro, 205479, 1000)); # export this }; route 92.118.30.16/28 via "br-internal"; @@ -809,7 +837,34 @@ in { route 92.118.30.6/31 via "wg-cofractal-gn"; }; protocol static export6 { - ipv6 {}; + ipv6 { + import filter { + if ((ro, 205479, 1000) ~ bgp_ext_community) then { + bgp_ext_community.add((ro, 205479, 2000)); # + bgp_ext_community.add((ro, 205479, 2001)); # + bgp_ext_community.add((ro, 205479, 2002)); # + bgp_ext_community.add((ro, 205479, 2003)); # + bgp_ext_community.add((ro, 205479, 3000)); # clouvider from clouvider-lon01 + bgp_ext_community.add((ro, 205479, 4000)); # frantech from frantech-nyc01/veloxserv from etheroute-lon01 + bgp_ext_community.add((ro, 205479, 4001)); # veloxserv from rexxar + bgp_ext_community.add((ro, 205479, 4002)); # mercury from etheroute-lon01 + bgp_ext_community.add((ro, 205479, 5000)); # linx route collector from rexxar + bgp_ext_community.add((ro, 205479, 5001)); # linx route server from rexxar + bgp_ext_community.add((ro, 205479, 5002)); # facebook from rexxar + bgp_ext_community.add((ro, 205479, 5003)); # openpeering from rexxar + bgp_ext_community.add((ro, 205479, 5004)); # freetransitnet from rexxar + bgp_ext_community.add((ro, 205479, 5005)); # he from rexxar + bgp_ext_community.add((ro, 205479, 5006)); # clouvider from rexxar + bgp_ext_community.add((ro, 205479, 5007)); # google from rexxar + bgp_ext_community.add((ro, 205479, 5008)); # cloudflare from rexxar + bgp_ext_community.add((ro, 205479, 5009)); # fastly from rexxar + bgp_ext_community.add((ro, 205479, 5010)); # ovh from rexxar + bgp_ext_community.add((ro, 205479, 6000)); # EMF: EMF-IX Route Server + bgp_ext_community.add((ro, 205479, 7000)); # 1299 from rexxar + } + accept; + }; + }; route ::/0 via 2a09:a442::1:2 bfd { # EE, cofractal-ams01 preference = 10; @@ -837,6 +892,7 @@ in { route 2a09:a443:2::/64 via "vl-eduroam"; route 2a09:a443:3::/48 via "vl-eduroam"; route 2a09:a443::/32 unreachable { + bgp_ext_community.add((ro, 205479, 10)); # export this - internal bgp_ext_community.add((ro, 205479, 1000)); # export this }; }; @@ -884,7 +940,7 @@ in { direct; bfd on; - ipv4 { + ipv6 { export filter { if source != RTS_STATIC then reject; if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; @@ -922,7 +978,7 @@ in { direct; bfd on; - ipv4 { + ipv6 { export filter { if source != RTS_STATIC then reject; if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; @@ -960,7 +1016,7 @@ in { direct; bfd on; - ipv4 { + ipv6 { export filter { if source != RTS_STATIC then reject; if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject; @@ -998,7 +1054,7 @@ in { direct; bfd on; - ipv4 { + ipv6 { export filter { if source != RTS_STATIC then reject; if ! ((ro, 205479, 10) ~ bgp_ext_community) then reject;