depot/ops/nixos/lib/blade-router.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ lib, config, pkgs, ... }:
with lib;
let
  cfg = config.my.blade-router;
in
{
  imports = [
    ../lib/bgp.nix
  ];

  options.my.blade-router = {
    addresses.linknet.v4 = {
      local = mkOption { type = types.str; };
      remote = mkOption { type = types.str; };
      prefixLength = mkOption { type = types.int; default = 31; };
    };
    addresses.linknet.v6 = {
      local = mkOption { type = types.str; };
      remote = mkOption { type = types.str; };
      prefixLength = mkOption { type = types.int; default = 126; };
    };

    addresses.br-public.v4 = {
      addr = mkOption { type = types.str; };
      prefixLength = mkOption { type = types.int; default = 24; };
    };
    addresses.br-public.v6 = {
      addr = mkOption { type = types.str; };
      prefixLength = mkOption { type = types.int; default = 48; };
    };

    addresses.br-public-vip.v4 = {
      addr = mkOption { type = types.str; default = "92.118.28.1"; };
      prefixLength = mkOption { type = types.int; default = 24; };
    };
    addresses.br-public-vip.v6 = {
      addr = mkOption { type = types.str; default = "2a09:a441::1"; };
      prefixLength = mkOption { type = types.int; default = 48; };
    };
    addresses.br-public-vip.v6-ll = {
      addr = mkOption { type = types.str; default = "fe80::f00f"; };
      prefixLength = mkOption { type = types.int; default = 64; };
    };

    addresses.br-public-radvd-prefix = {
      addr = mkOption { type = types.str; default = "2a09:a441:0:ffff::"; };
      prefixLength = mkOption { type = types.int; default = 64; };
    };

    linx.enable = mkEnableOption "linx";
    addresses.vl-linx.v4 = {
      addr = mkOption { type = types.str; default = "195.66.224.58"; };
      prefixLength = mkOption { type = types.int; default = 21; };
    };
    addresses.vl-linx.v6 = {
      addr = mkOption { type = types.str; default = "2001:7f8:4::3:22a7:1"; };
      prefixLength = mkOption { type = types.int; default = 64; };
    };

    vrrp.priority = mkOption { type = types.int; };
  };

  config = {
    boot.kernel.sysctl = {
      "net.ipv4.ip_forward" = 1;
      "net.ipv6.conf.all.forwarding" = 1;

      "net.ipv4.conf.vl-linx.arp_announce" = 1;
      "net.ipv4.conf.vl-linx.arp_ignore" = 1;
      "net.ipv4.neigh.vl-linx.base_reachable_time_ms" = 14400000;
      "net.ipv6.neigh.vl-linx.base_reachable_time_ms" = 14400000;
    };

    networking = {
      vlans.vl-transit = {
        id = 100;
        interface = "en-internet";
      };
      vlans.vl-linx = {
        id = 200;
        interface = "en-internet";
      };

      interfaces.br-public.ipv4.addresses = [{
        address = config.my.blade-router.addresses.br-public.v4.addr;
        prefixLength = config.my.blade-router.addresses.br-public.v4.prefixLength;
      }];
      interfaces.br-public.ipv6.addresses = [{
        address = config.my.blade-router.addresses.br-public.v6.addr;
        prefixLength = config.my.blade-router.addresses.br-public.v6.prefixLength;
      }];
      interfaces.vl-transit.ipv4.addresses = [{
        address = config.my.blade-router.addresses.linknet.v4.local;
        prefixLength = config.my.blade-router.addresses.linknet.v4.prefixLength;
      }];
      interfaces.vl-transit.ipv6.addresses = [{
        address = config.my.blade-router.addresses.linknet.v6.local;
        prefixLength = config.my.blade-router.addresses.linknet.v6.prefixLength;
      }];
      interfaces.vl-linx.macAddress = "e4:11:5b:ac:e4:00";
      interfaces.vl-linx.ipv4.addresses = [{
        address = cfg.addresses.vl-linx.v4.addr;
        prefixLength = cfg.addresses.vl-linx.v4.prefixLength;
      }];
      interfaces.vl-linx.ipv6.addresses = [{
        address = cfg.addresses.vl-linx.v6.addr;
        prefixLength = cfg.addresses.vl-linx.v6.prefixLength;
      }];
      defaultGateway = config.my.blade-router.addresses.linknet.v4.remote;
      defaultGateway6 = config.my.blade-router.addresses.linknet.v6.remote;
      firewall.extraCommands = ''
        iptables -A INPUT -p vrrp -i br-mgmt -j ACCEPT
        ip6tables -A INPUT -p vrrp -i br-mgmt -j ACCEPT
      '';
    };

    services.lukegbgp = {
      enable = true;
      config = let
        linx = {
          local = {
            asn = 205479;
            v4 = cfg.addresses.vl-linx.v4.addr;
            v6 = cfg.addresses.vl-linx.v6.addr;
          };
        };
      in {
        local.routerID = config.my.blade-router.addresses.linknet.v4.local;
        peering.veloxserv = {
          local = {
            asn = 205479;
            v4 = config.my.blade-router.addresses.linknet.v4.local;
            v6 = config.my.blade-router.addresses.linknet.v6.local;
          };
          remote = {
            asn = 3170;
            export_community = 4001;
            bgp_local_pref = 101;
            drop_asns = [
              15169  # prefer RS to transit
            ];
            routers = [{
              v4 = config.my.blade-router.addresses.linknet.v4.remote;
              v6 = config.my.blade-router.addresses.linknet.v6.remote;
            }];
          };
        };
        peering.bgptoolscollector = {
          local = {
            asn = 205479;
            v4 = config.my.blade-router.addresses.linknet.v4.local;
            v6 = config.my.blade-router.addresses.linknet.v6.local;
          };
          remote = {
            asn = 212232;
            export_community = 5000;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "185.230.223.42";
              v6 = "2a0c:2f07:9459::b1";
              multihop = 64;
            }];
            prefix_limit.v4 = 0;
            prefix_limit.v6 = 0;
            is_route_collector = true;
          };
        };
        peering.linxcollector = linx // {
          remote = {
            asn = 5459;
            export_community = 5000;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.224.254";
              v6 = "2001:7f8:4::1553:1";
            }];
            prefix_limit.v4 = 0;
            prefix_limit.v6 = 0;
            is_route_collector = true;
          };
        };
        peering.linx = linx // {
          remote = {
            asn = 8714;
            export_community = 5001;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.230";
              v6 = "2001:7f8:4::220a:1";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.225.231";
              v6 = "2001:7f8:4::220a:2";
            }];
            bgp_local_pref = 109;
            must_be_next_hop = false;
          };
        };
        peering.facebook = linx // {
          remote = {
            asn = 32934;
            export_community = 5002;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.69";
              v6 = "2001:7f8:4::80a6:1";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.225.121";
              v6 = "2001:7f8:4::80a6:2";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.227.19";
              v6 = "2001:7f8:4::80a6:5";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.226.140";
              v6 = "2001:7f8:4::80a6:3";
            }];
            bgp_local_pref = 120;
            prefix_limit.v4 = 100;
            prefix_limit.v6 = 100;
          };
        };
        peering.openpeering = linx // {
          remote = {
            asn = 20562;
            export_community = 5003;
            passive = true;  # pending
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.53";
              v6 = "2001:7f8:4::5052:1";
            }];
            bgp_local_pref = 110;
            prefix_limit.v4 = 16000;
            prefix_limit.v6 = 3000;
          };
        };
        peering.freetransitnet = linx // {
          remote = {
            asn = 212895;
            export_community = 5004;
            passive = true;  # pending v6
            bgp_local_pref = 100;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.105";
              v6 = "2001:7f8:4::3:3f9f:2";
            }];
          };
        };
        peering.he = linx // {
          remote = {
            asn = 6939;
            export_community = 5005;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.224.21";
              v6 = "2001:7f8:4::1b1b:1";
            }];
            bgp_local_pref = 108;
            prefix_limit.v4 = 176000;
            prefix_limit.v6 = 156000;
          };
        };
        peering.clouvider = linx // {
          remote = {
            asn = 62240;
            export_community = 5006;
            passive = true;  # pending
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.227.14";
              v6 = "2001:7f8:4::f320:1";
            }];
            bgp_local_pref = 120;
            prefix_limit.v4 = 1000;
            prefix_limit.v6 = 1000;
          };
        };
        peering.google = linx // {
          remote = {
            asn = 15169;
            export_community = 5007;
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.224.125";
              v6 = "2001:7f8:4::3b41:1";
            }];
            bgp_local_pref = 120;
            prefix_limit.v4 = 15000;
            prefix_limit.v6 = 10000;
          };
        };
        peering.cloudflare = linx // {
          remote = {
            asn = 13337;
            export_community = 5008;
            passive = true;  # pending
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.179";
              v6 = "2001:7f8:4::3417:1";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.227.207";
              v6 = "2001:7f8:4::3417:2";
            }];
            bgp_local_pref = 120;
            prefix_limit.v4 = 20000;
            prefix_limit.v6 = 2000;
          };
        };
        peering.fastly = linx // {
          remote = {
            asn = 54113;
            export_community = 5009;
            passive = true;  # pending
            routers = [{
              enabled = cfg.linx.enable;
              v4 = "195.66.225.91";
              v6 = "2001:7f8:4::d361:1";
            } {
              enabled = cfg.linx.enable;
              v4 = "195.66.227.114";
              v6 = "2001:7f8:4::d361:2";
            }];
            bgp_local_pref = 120;
            prefix_limit.v4 = 250;
            prefix_limit.v6 = 250;
          };
        };
        export.v4 = [ "92.118.28.0/24" ];
        export.v6 = [ "2a09:a441::/32" ];
      };
    };

    services.keepalived = let
      mgmtBase = {
        interface = "br-mgmt";
        state = "MASTER";
        priority = config.my.blade-router.vrrp.priority;
      };
    in {
      enable = true;
      vrrpInstances.mgmtGateway4 = mgmtBase // {
        virtualIps = [
          { addr = "10.100.0.1/23"; }
          { addr = "${config.my.blade-router.addresses.br-public-vip.v4.addr}/${toString config.my.blade-router.addresses.br-public-vip.v4.prefixLength}"; dev = "br-public"; }
        ];
        virtualRouterId = 1;
      };
      vrrpInstances.mgmtGateway6 = mgmtBase // {
        virtualIps = [
          { addr = "${config.my.blade-router.addresses.br-public-vip.v6-ll.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6-ll.prefixLength}"; dev = "br-public"; }
          { addr = "${config.my.blade-router.addresses.br-public-vip.v6.addr}/${toString config.my.blade-router.addresses.br-public-vip.v6.prefixLength}"; dev = "br-public"; }
        ];
        virtualRouterId = 2;
      };
      extraGlobalDefs = ''
        enable_script_security
        script_user root
      '';
      extraConfig = ''
        vrrp_sync_group mgmtGateway {
          group {
            mgmtGateway4
            mgmtGateway6
          }
        }
      '';
    };

    services.radvd = {
      enable = true;
      config = ''
        interface br-public {
          AdvSendAdvert on;
          MinRtrAdvInterval 30;
          MaxRtrAdvInterval 100;
          AdvRASrcAddress {
            ${config.my.blade-router.addresses.br-public-vip.v6-ll.addr};
          };
          prefix ${config.my.blade-router.addresses.br-public-radvd-prefix.addr}/${toString config.my.blade-router.addresses.br-public-radvd-prefix.prefixLength} {
            AdvOnLink on;
            AdvAutonomous on;
            AdvRouterAddr off;
          };
        };
      '';
    };
  };
}