depot/ops/nixos/blade-tuvok/default.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, config, ... }:
let
  inherit (depot.ops) secrets;
in {
  imports = [
    ../lib/blade-router.nix
    ../lib/blade.nix
    ../lib/fup.nix
  ];

  boot.loader.grub.device = "/dev/disk/by-id/usb-SanDisk_Cruzer_Blade_00023903100122082002-0:0";

  # Networking!
  networking = {
    hostName = "blade-tuvok";
    hostId = "525229f7";
    firewall.allowedTCPPorts = [ 80 443 ];
    firewall.allowedUDPPorts = [
      # Wireguard
      51820 51821 51822
    ];
    localCommands = ''
      # Check if we already have our little minicontainer setup
      ip netns list | grep -q wg-endpoint || (
        ip netns add wg-endpoint
        ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint
        ip link set dev ve-wg-endpoint master br-public
        ip link set dev ve-wg-endpoint up
        ip -n wg-endpoint link set dev ve-int up
        ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int
        ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int
        ip -n wg-endpoint route add default via 92.118.28.1
        ip -n wg-endpoint route add default via 2a09:a441::1
        ip netns exec wg-endpoint ${pkgs.procps}/bin/sysctl net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.default.forwarding=1
      )

      # We are forcing the MAC on vl-linx to be something particular,
      # but we also need to add the corresponding LL address.
      # This is cursed.
      ip -6 addr add fe80::e611:5bff:feac:e400/64 dev vl-linx &>/dev/null || true
    '';
    nat.extraCommands = ''
      iptables -w -t nat -A nixos-nat-post -m mark --mark 0x1 -o vl-linx -j SNAT --to-source 92.118.28.1
    '';
  };
  my.ip.tailscale = "100.119.123.33";
  my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6277:7b21";
  my.blade.bay = 6;
  my.blade.macAddress = {
    internal = "e4:11:5b:ac:e3:fe";
    storage = "e4:11:5b:ac:e4:02";
    internet = "e4:11:5b:ac:e3:ff";
    public = "e4:11:5b:ac:e4:00";
  };

  services.ceph = {
    mon.enable = true;
    osd = {
      enable = true;
      daemons = [ "3" ];
    };
  };

  services.nginx = {
    enable = true;
    recommendedTlsSettings = true;
    recommendedGzipSettings = true;
    virtualHosts."objdump.zxcvbnm.ninja" = {
      default = true;
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://localhost:7480";
        extraConfig = ''
          proxy_redirect off;
          client_max_body_size 0;
          proxy_buffering off;
        '';
      };
    };
  };
  my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = {
    hostnames = [ "objdump.zxcvbnm.ninja" "*.objdump.zxcvbnm.ninja" ];
    nginxVirtualHosts = [ "objdump.zxcvbnm.ninja" ];
  };
  my.fup.listen = [
    "0.0.0.0" "[::]"
  ];

  my.blade-router = {
    addresses.linknet = {
      v4 = { local = "195.74.55.21"; remote = "195.74.55.20"; };
      v6 = {
        local = "2a03:ee40:8080:9:1::2";
        remote = "2a03:ee40:8080:9:1::1";
      };
    };

    addresses.br-public = {
      v4.addr = "92.118.28.254";
      v6.addr = "2a09:a441::ffff";
    };

    linx.enable = true;

    vrrp.priority = 100;
  };

  networking.wireguard = let
    ifBase = {
      listenPort = null;
      allowedIPsAsRoutes = false;
      socketNamespace = "wg-endpoint";
    };
    peerBase.allowedIPs = [
      "0.0.0.0/0"
      "::/0"
    ];
    swannPeerBase = peerBase // {
      endpoint = null;  # dynamic
      publicKey = secrets.wireguard.tuvok-swann.swann.publicKey;
    };
    swannBase = ifBase // {
      privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey;
      peers = [swannPeerBase];
    };
  in {
    enable = true;
    interfaces.wg-swann-ee = swannBase // {
      ips = [
        "2a09:a442::2:2/112"
        "92.118.30.3/31"
      ];
      listenPort = 51821;
    };
    interfaces.wg-swann-gnet = swannBase // {
      ips = [
        "2a09:a442::3:2/112"
        "92.118.30.5/31"
      ];
      listenPort = 51822;
      peers = [(swannPeerBase // {
        endpoint = "185.250.189.204:51822";
      })];
    };
  };
  environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile {
    name = "bird-wg-endpoint.conf";
    text = ''
      router id 92.118.28.252;
      protocol kernel {
        persist;
        ipv4 {
          import none;
          export all;
        };
      };
      protocol kernel {
        persist;
        ipv6 {
          import none;
          export all;
        };
      };
      protocol device {};

      protocol static export4 {
        ipv4 {};
        route 92.118.30.0/24 via 92.118.30.2 bfd {
          # EE
          preference = 10;
        };
        route 92.118.30.0/24 via 92.118.30.4 bfd {
          # GNetwork
          preference = 200;
        };
      };
      protocol static export6 {
        ipv6 {};
        route 2a09:a443::/32 via 2a09:a442::1:1 bfd {
          # Virgin Media
          preference = 100;
        };
        route 2a09:a443::/32 via 2a09:a442::2:1 bfd {
          # EE
          preference = 10;
        };
        route 2a09:a443::/32 via 2a09:a442::3:1 bfd {
          # GNetwork
          preference = 200;
        };
      };

      protocol bfd {
        interface "*" {
          min rx interval 10ms;
          min tx interval 50ms;
          idle tx interval 1s;
          multiplier 20;
        };
        neighbor 92.118.30.2;
        neighbor 2a09:a442::2:1;
        neighbor 92.118.30.4;
        neighbor 2a09:a442::3:1;
      };
    '';
    checkPhase = ''
      ${pkgs.bird2}/bin/bird -d -p -c $out
    '';
  };
  systemd.services.bird-wg-endpoint = {
    wantedBy = [ "multi-user.target" ];
    reloadIfChanged = true;

    description = "BIRD inside wg-endpoint netns";
    after = [ "network.target" ];
    restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ];

    serviceConfig = {
      Type = "forking";
      Restart = "on-failure";
      CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID"
                                # see bird/sysdep/linux/syspriv.h
                                "CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ];
      ProtectSystem = "full";
      ProtectHome = "yes";
      SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io";
      MemoryDenyWriteExecute = "yes";

      ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down";
      ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl";
      ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'";
      NetworkNamespacePath = "/var/run/netns/wg-endpoint";
    };
  };
  # It is at this point he realises it would've been easier if he'd used NixOS containers instead.
  systemd.services.bird-wg-endpoint-exporter = {
    after = [ "bird-wg-endpoint.service" "network.target" ];
    wantedBy = [ "multi-user.target" ];
    serviceConfig = {
      DynamicUser = true;
      User = "bird-exporter";
      Group = "bird-exporter";
      PrivateTmp = true;
      Restart = "always";
      WorkingDirectory = "/tmp";
      SupplementaryGroups = "bird2";
      ExecStart = ''
        ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \
          -web.listen-address 0.0.0.0:9325 \
          -bird.socket /var/run/bird-wg-endpoint.ctl \
          -bird.v2=true \
          -format.new=true
      '';
    };
  };
  my.prometheus.additionalExporterPorts.bird-wg-endpoint-exporter = 9325;
  services.lukegbgp.config.export = {
    v4Extra = ''
      route 92.118.30.0/24 via 92.118.28.252;
    '';
    v6Extra = ''
      route 2a09:a443::/32 via 2a09:a441::f00f;
    '';
  };
  systemd.services.wireguard-wg-swann-ee.after = lib.mkAfter [ "network-local-commands.service" ];
  systemd.services.wireguard-wg-swann-gnet.after = lib.mkAfter [ "network-local-commands.service" ];

}