# SPDX-FileCopyrightText: 2020 Luke Granger-Brown # # SPDX-License-Identifier: Apache-2.0 { depot, lib, pkgs, config, ... }: let inherit (depot.ops) secrets; in { imports = [ ../lib/blade-router.nix ../lib/blade.nix ../lib/fup.nix ]; boot.loader.grub.device = "/dev/disk/by-id/usb-USB_SanDisk_3.2Gen1_0101cabb1ebdbdc0fd7b18edd207d43717c39c4a59d1b138b363e315841eca15743400000000000000000000443273100087260091558107b6a8e06e-0:0"; # Networking! networking = { hostName = "blade-tuvok"; hostId = "525229f7"; firewall.allowedTCPPorts = [ 80 443 ]; firewall.allowedUDPPorts = [ # Wireguard 51820 51821 51822 ]; localCommands = '' # Check if we already have our little minicontainer setup ip netns list | grep -q wg-endpoint || ( ip netns add wg-endpoint ip link add ve-wg-endpoint type veth peer name ve-int netns wg-endpoint ip link set dev ve-wg-endpoint master br-public ip link set dev ve-wg-endpoint up ip -n wg-endpoint link set dev ve-int up ip -n wg-endpoint addr add 92.118.28.252/24 dev ve-int ip -n wg-endpoint addr add 2a09:a441::f00f/48 dev ve-int ip -n wg-endpoint route add default via 92.118.28.1 ip -n wg-endpoint route add default via 2a09:a441::1 ip netns exec wg-endpoint ${pkgs.procps}/bin/sysctl net.ipv4.ip_forward=1 net.ipv6.conf.all.forwarding=1 net.ipv6.conf.default.forwarding=1 ) # We are forcing the MAC on vl-linx to be something particular, # but we also need to add the corresponding LL address. # This is cursed. ip -6 addr add fe80::e611:5bff:feac:e400/64 dev vl-linx &>/dev/null || true ''; nat.extraCommands = '' iptables -w -t nat -A nixos-nat-post -m mark --mark 0x1 -o vl-linx -j SNAT --to-source 92.118.28.1 ''; }; my.ip.tailscale = "100.119.123.33"; my.blade.bay = 6; my.blade.macAddress = { internal = "e4:11:5b:ac:e3:fe"; storage = "e4:11:5b:ac:e4:02"; internet = "e4:11:5b:ac:e3:ff"; public = "e4:11:5b:ac:e4:00"; }; services.ceph = { mon.enable = true; osd = { enable = true; daemons = [ "3" ]; }; }; services.nginx = { enable = true; recommendedTlsSettings = true; recommendedGzipSettings = true; virtualHosts."objdump.zxcvbnm.ninja" = { default = true; forceSSL = true; locations."/" = { proxyPass = "http://localhost:7480"; extraConfig = '' proxy_redirect off; client_max_body_size 0; proxy_buffering off; ''; }; }; }; my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = { extraNames = [ "*.objdump.zxcvbnm.ninja" ]; nginxVirtualHosts = [ "objdump.zxcvbnm.ninja" ]; }; my.fup.listen = [ "0.0.0.0" "[::]" ]; my.blade-router = { addresses.linknet = { v4 = { local = "195.74.55.21"; remote = "195.74.55.20"; }; v6 = { local = "2a03:ee40:8080:9:1::2"; remote = "2a03:ee40:8080:9:1::1"; }; }; addresses.br-public = { v4.addr = "92.118.28.254"; v6.addr = "2a09:a441::ffff"; }; linx.enable = true; vrrp.priority = 100; }; networking.wireguard = let ifBase = { listenPort = null; allowedIPsAsRoutes = false; socketNamespace = "wg-endpoint"; }; peerBase.allowedIPs = [ "0.0.0.0/0" "::/0" ]; swannPeerBase = peerBase // { endpoint = null; # dynamic publicKey = secrets.wireguard.tuvok-swann.swann.publicKey; }; swannBase = ifBase // { privateKey = secrets.wireguard.tuvok-swann.tuvok.privateKey; peers = [swannPeerBase]; }; in { enable = true; interfaces.wg-swann-vm = swannBase // { ips = [ "2a09:a442::1:2/112" "92.118.30.1/31" ]; listenPort = 51820; }; interfaces.wg-swann-ee = swannBase // { ips = [ "2a09:a442::2:2/112" "92.118.30.3/31" ]; listenPort = 51821; }; interfaces.wg-swann-gnet = swannBase // { ips = [ "2a09:a442::3:2/112" "92.118.30.5/31" ]; listenPort = 51822; peers = [(swannPeerBase // { endpoint = "185.250.189.204:51822"; })]; }; }; environment.etc."bird/bird-wg-endpoint.conf".source = pkgs.writeTextFile { name = "bird-wg-endpoint.conf"; text = '' router id 92.118.28.252; protocol kernel { persist; ipv4 { import none; export all; }; }; protocol kernel { persist; ipv6 { import none; export all; }; }; protocol device {}; protocol static export4 { ipv4 {}; route 92.118.30.0/24 via 92.118.30.0 bfd { # Virgin Media preference = 100; }; route 92.118.30.0/24 via 92.118.30.2 bfd { # EE preference = 10; }; route 92.118.30.0/24 via 92.118.30.4 bfd { # GNetwork preference = 200; }; }; protocol static export6 { ipv6 {}; route 2a09:a443::/32 via 2a09:a442::1:1 bfd { # Virgin Media preference = 100; }; route 2a09:a443::/32 via 2a09:a442::2:1 bfd { # EE preference = 10; }; route 2a09:a443::/32 via 2a09:a442::3:1 bfd { # GNetwork preference = 200; }; }; protocol bfd { interface "*" { min rx interval 10ms; min tx interval 50ms; idle tx interval 1s; multiplier 20; }; neighbor 92.118.30.0; neighbor 2a09:a442::1:1; neighbor 92.118.30.2; neighbor 2a09:a442::2:1; neighbor 92.118.30.4; neighbor 2a09:a442::3:1; }; ''; checkPhase = '' ${pkgs.bird2}/bin/bird -d -p -c $out ''; }; systemd.services.bird-wg-endpoint = { wantedBy = [ "multi-user.target" ]; reloadIfChanged = true; description = "BIRD inside wg-endpoint netns"; after = [ "network.target" ]; restartTriggers = [ config.environment.etc."bird/bird-wg-endpoint.conf".source ]; serviceConfig = { Type = "forking"; Restart = "on-failure"; CapabilityBoundingSet = [ "CAP_CHOWN" "CAP_FOWNER" "CAP_DAC_OVERRIDE" "CAP_SETUID" "CAP_SETGID" # see bird/sysdep/linux/syspriv.h "CAP_NET_BIND_SERVICE" "CAP_NET_BROADCAST" "CAP_NET_ADMIN" "CAP_NET_RAW" ]; ProtectSystem = "full"; ProtectHome = "yes"; SystemCallFilter="~@cpu-emulation @debug @keyring @module @mount @obsolete @raw-io"; MemoryDenyWriteExecute = "yes"; ExecStop = "${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl down"; ExecStart = "${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -u bird2 -g bird2 -s /var/run/bird-wg-endpoint.ctl"; ExecReload = "/bin/sh -c '${pkgs.bird2}/bin/bird -c /etc/bird/bird-wg-endpoint.conf -p && ${pkgs.bird2}/bin/birdc -s /var/run/bird-wg-endpoint.ctl configure'"; NetworkNamespacePath = "/var/run/netns/wg-endpoint"; }; }; # It is at this point he realises it would've been easier if he'd used NixOS containers instead. systemd.services.bird-wg-endpoint-exporter = { after = [ "bird-wg-endpoint.service" "network.target" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { DynamicUser = true; User = "bird-exporter"; Group = "bird-exporter"; PrivateTmp = true; Restart = "always"; WorkingDirectory = "/tmp"; SupplementaryGroups = "bird2"; ExecStart = '' ${depot.pkgs.prometheus-bird-exporter-lfty}/bin/bird_exporter \ -web.listen-address 0.0.0.0:9325 \ -bird.socket /var/run/bird-wg-endpoint.ctl \ -bird.v2=true \ -format.new=true ''; }; }; my.prometheus.additionalExporterPorts.bird-wg-endpoint-exporter = 9325; services.lukegbgp.config.export = { v4Extra = '' route 92.118.30.0/24 via 92.118.28.252; ''; v6Extra = '' route 2a09:a443::/32 via 2a09:a441::f00f; ''; }; }