depot/ops/nixos/rexxar/default.nix

# SPDX-FileCopyrightText: 2024 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, config, ... }:
{
  imports = [
    ../lib/zfs.nix
    ./bgp.nix
    ../lib/bgp.nix
    ../lib/gitlab-runner-cacher.nix
    #../lib/nixbuild-distributed.nix  # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid
    ../lib/hackyplayer.nix
    ../lib/emfminiserv.nix
    ../lib/seaweedfs.nix
    ../lib/fup.nix
    ./bsky-pds.nix
  ];

  # Otherwise _this_ machine won't enumerate things properly.
  boot.zfs.devNodes = "/dev/disk/by-id";

  boot.initrd = {
    availableKernelModules = [
      "nvme"
      "xhci_pci"
      "ahci"
      "usb_storage"
      "usbhid"
      "sd_mod"
      "sr_mod"
    ];
    systemd.enable = true;
  };
  security.tpm2.enable = true;
  boot.kernelModules = [ "kvm-amd" ];
  hardware.cpu.amd.updateMicrocode = true;
  boot.kernelParams = [
    "nomodeset"
  ];
  environment.systemPackages = with pkgs; [
    clevis
  ];

  # Use the systemd-boot EFI boot loader.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";

  fileSystems = let
    zfs = device: {
      device = device;
      fsType = "zfs";
    };
  in {
    "/" = zfs "zboot/local/root";
    "/nix" = zfs "zboot/local/nix";
    "/persist" = zfs "zboot/safe/persist";

    "/store" = zfs "zu2/safe/store";
    "/home" = zfs "zu2/safe/home";

    "/store/emf" = zfs "zu2/safe/store/emf";
    "/store/emf/2024" = zfs "zu2/safe/store/emf/2024";
    "/store/emf/2024/video" = zfs "zu2/safe/store/emf/2024/video";
    "/store/emf/2024/video/source" = zfs "zu2/safe/store/emf/2024/video/source";
    "/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input";
    "/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output";

    "/var/lib/private/seaweedfs-master" = zfs "zu2/safe/var/lib/private/seaweedfs-master";
    "/var/lib/private/seaweedfs-filer" = zfs "zu2/safe/var/lib/private/seaweedfs-filer";
    "/var/lib/private/seaweedfs-volume" = zfs "zu2/safe/var/lib/private/seaweedfs-volume";
    "/var/lib/private/seaweedfs-volume/data" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/data";
    "/var/lib/private/seaweedfs-volume/idx" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/idx";

    "/boot" = {
      device = "/dev/disk/by-label/ESP";
      fsType = "vfat";
    };
    "/boot2" = {
      device = "/dev/disk/by-label/ESP2";
      fsType = "vfat";
    };
  };
  boot.loader.systemd-boot.extraInstallCommands = ''
    rsync -a /boot/ /boot2/
  '';

  nix.settings.max-jobs = lib.mkDefault 64;

  # Networking!
  networking = {
    hostName = "rexxar";
    domain = "as205479.net";
    hostId = "b46c2ae9";
    useNetworkd = true;
    firewall = {
      allowedUDPPorts = [
        51821 51822 51823
        34197  # factorio
        443
        3784  # BFD
      ];
      allowedTCPPorts = [
        80 443
      ];
    };
  };
  systemd.network = let
    wireguard = { name, listenPort, privateKey, publicKey, endpoint ? null }: {
      netdevConfig = {
        Name = name;
        Kind = "wireguard";
        Description = "WireGuard tunnel ${name}";
      };
      wireguardConfig = {
        ListenPort = listenPort;
        PrivateKeyFile = privateKey;
      };
      wireguardPeers = [(lib.mkMerge [{
        PublicKey = publicKey;
        AllowedIPs = [
          "0.0.0.0/0"
          "::/0"
        ];
      } (lib.mkIf (endpoint != null) {
        Endpoint = endpoint;
      })])];
    };
    swannWireguard = args: wireguard (args // {
      privateKey = config.my.vault.secrets.wg-swann-private.path;
      publicKey = "XyfovUP6GUwIg15t5UWxicfxooeto/U/7nLs7Zu8HH4=";
    });
    cofractalWireguard = args: wireguard (args // {
      privateKey = config.my.vault.secrets.wg-cofractal-ams01-private.path;
      publicKey = "qKi6mWIhV2n16LGH16Iug5W+Bx4Fx7eprxCgA/1Ra1g=";
    });
  in {
    netdevs."40-wg-swann-ee" = swannWireguard {
      name = "wg-swann-ee";
      listenPort = 51821;
    };
    netdevs."40-wg-swann-gnet" = swannWireguard {
      name = "wg-swann-gnet";
      listenPort = 51822;
      endpoint = "185.250.189.20:51822";
    };
    netdevs."40-wg-cofractal" = cofractalWireguard {
      name = "wg-cofractal";
      listenPort = 51823;
      endpoint = "[2a09:a446:1337:ffff::10]:51823";
    };

    networks."40-wg-swann-ee" = {
      matchConfig.Name = "wg-swann-ee";
      address = [
        "92.118.30.3/31"
        "2a09:a442::2:2/112"
      ];
    };
    networks."40-wg-swann-gnet" = {
      matchConfig.Name = "wg-swann-gnet";
      address = [
        "92.118.30.5/31"
        "2a09:a442::3:2/112"
      ];
    };
    networks."40-wg-cofractal" = {
      matchConfig.Name = "wg-cofractal";
      address = [
        "169.254.200.0/31"
      ];
    };

    networks."10-usb0" = {
      matchConfig.Name = "usb0";
      address = [
        "169.254.0.1/24"
        # IPMI 169.254.0.17
      ];
    };
    networks."10-enp193s0f0np0" = {
      matchConfig.Name = "enp193s0f0np0";
      # Telia
      address = [
        "62.115.150.105/31"
        "2001:2035:0:1f7e::2/126"
      ];
    };
    networks."10-enp193s0f1np1" = {
      matchConfig.Name = "enp193s0f1np1";
      networkConfig.VLAN = [ "vl-velox1" ];
    };
    # enp9s0f0 - previously velox copper cable 1, now enp193s0f1np1 via 10G
    networks."10-enp9s0f1" = {
      matchConfig.Name = "enp9s0f1";
      networkConfig.VLAN = [ "vl-velox2" "vl-linx" ]; # also IPMI, vlan 300
    };
    netdevs."20-vl-velox1" = {
      netdevConfig = {
        Name = "vl-velox1";
        Kind = "vlan";
        MACAddress = "8C:1F:64:0B:6F:00";
      };
      vlanConfig = {
        Id = 100;
      };
    };
    networks."20-vl-velox1" = {
      matchConfig.Name = "vl-velox1";
      address = [
        "195.74.55.21/31"
        "2a03:ee40:8080:9:1::2/126"
      ];
      networkConfig.DNS = [
        "2001:4860:4860::8888"
        "2001:4860:4860::8844"
        "8.8.8.8"
        "8.8.4.4"
        "1.1.1.1"
      ];
      networkConfig.DNSDefaultRoute = true;
      routes = [{
        Gateway = "195.74.55.20";
      } {
        Gateway = "2a03:ee40:8080:9:1::1";
      }];
    };
    netdevs."20-vl-velox2" = {
      netdevConfig = {
        Name = "vl-velox2";
        Kind = "vlan";
        MACAddress = "8C:1F:64:0B:6F:01";
      };
      vlanConfig = {
        Id = 100;
      };
    };
    networks."20-vl-velox2" = {
      matchConfig.Name = "vl-velox2";
      address = [
        "195.74.55.23/31"
        "2a03:ee40:8080:9:2::2/126"
      ];
      networkConfig.DNS = [
        "2001:4860:4860::8888"
        "2001:4860:4860::8844"
        "8.8.8.8"
        "8.8.4.4"
        "1.1.1.1"
      ];
      networkConfig.DNSDefaultRoute = true;
      routes = [{
        Gateway = "195.74.55.22";
      } {
        Gateway = "2a03:ee40:8080:9:2::1";
      }];
    };
    netdevs."20-vl-linx" = {
      netdevConfig = {
        Name = "vl-linx";
        Kind = "vlan";
        MACAddress = "8C:1F:64:0B:6F:02";
      };
      vlanConfig = {
        Id = 200;
      };
    };
    networks."20-vl-linx" = {
      matchConfig.Name = "vl-linx";
      address = [
        "195.66.224.58/21"
        "2001:7f8:4::3:22a7:1/48"
      ];
      networkConfig = {
        IPv6LinkLocalAddressGenerationMode = "eui64";
        LLMNR = false;
        MulticastDNS = false;
        IPv6AcceptRA = false;
        IPv4ProxyARP = false;
        IPv6ProxyNDP = false;
        IPv6SendRA = false;
      };
    };
    networks."60-lo" = {
      matchConfig.Name = "lo";
      addresses = [{
        Address = "127.0.0.1/8";
        Scope = "host";
      } {
        Address = "::1/128";
      } {
        Address = "92.118.30.251/32";
      } {
        Address = "2a09:a442:1000::/128";
      }];
    };
  };
  my.ip.tailscale = "100.97.110.48";
  my.ip.tailscale6 = "fd7a:115c:a1e0::3a01:6e30";
  #my.coredns.bind = [ "bond0" "tailscale0" "127.0.0.1" "::1" ];

  services.openssh.hostKeys = [
    {
      path = "/persist/etc/ssh/ssh_host_ed25519_key";
      type = "ed25519";
    }
    {
      path = "/persist/etc/ssh/ssh_host_rsa_key";
      type = "rsa";
      bits = 4096;
    }
  ];

  systemd.mounts = let
    bindMount' = dir: {
      unitConfig.RequiresMountsFor = dir;
      options = "bind";
      what = "/persist${dir}";
      where = dir;
    };
    bindMountSvc = dir: svc: (bindMount' dir) // {
      requiredBy = [svc];
      before = [svc];
      wantedBy = ["multi-user.target"];
    };
    bindMountSvcDynamic = dir: svc: (bindMount' "/var/lib/private/${dir}") // {
      requiredBy = [svc];
      before = [svc];
      wantedBy = ["multi-user.target"];
    };
    bindMount = dir: (bindMount' dir) // {
      wantedBy = ["multi-user.target"];
    };
  in [
    (bindMountSvc "/var/lib/tailscale" "tailscaled.service")
    (bindMountSvc "/var/lib/libvirt" "libvirt.service")
    (bindMountSvc "/var/lib/private/factorio" "factorio.service")
  ];

  boot.kernel.sysctl = {
    "net.ipv4.ip_forward" = 1;
    "net.ipv6.conf.all.forwarding" = 1;

    "net.ipv4.conf.vl-linx.arp_announce" = 1;
    "net.ipv4.conf.vl-linx.arp_ignore" = 1;
    "net.ipv4.neigh.vl-linx.base_reachable_time_ms" = 14400000;
    "net.ipv6.neigh.vl-linx.base_reachable_time_ms" = 14400000;

    "net.ipv4.neigh.default.gc_thresh1" = 2048;
    "net.ipv6.neigh.default.gc_thresh1" = 2048;
    "net.ipv4.neigh.default.gc_thresh2" = 4096;
    "net.ipv6.neigh.default.gc_thresh2" = 4096;
    "net.ipv4.neigh.default.gc_thresh3" = 8192;
    "net.ipv6.neigh.default.gc_thresh3" = 8192;
  };

  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];

  my.vault.secrets = let 
    wireguardSecret = key: {
      group = "systemd-network";
      template = ''
        {{- with secret "kv/apps/wireguard/rexxar" -}}
        {{- .Data.data.${key} -}}
        {{- end -}}
      '';
    };
  in {
    wg-swann-private = wireguardSecret "privateKeyToSwann";
    wg-cofractal-ams01-private = wireguardSecret "privateKeyToCofractalAms01";
  };

  users.users.lukegb.extraGroups = lib.mkAfter [ "hackyplayer" ];
  users.users.samw = {
    isNormalUser = true;
    extraGroups = lib.mkAfter [ "hackyplayer" ];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDQdGzwYiallvWXIHgSAf2GOwMJKA8bxPmwyuO+vsd1HwB65hMRPCpKS+FNLIpkrADNnuhGS3xGCGSSuQ+zAu/g="
      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNLrJyUXmiFWb9vhlTWZLYr64IsKut+c9TGqq3/uwPDeF4X0Qb2jzxqXfQcDSztjR09JHbC8BOqfpYYT9LHahIo="
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOOicoYtzJ3vXSkY7MkriQrfyZ/gP8ONM2OHhO0zi2cl"
      "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGUoi76VYnqsPZ4IjyfNb8NfaaAfcU9iAbARFEfYTuNwxmYUjv+RpCBC1Gqi0YpdONyrWi9XljjAtY92P4jZffQ="
    ];
  };

  services.factorio = {
    inherit (depot.ops.secrets.factorio) username token;
    enable = true;
    saveName = "lukegb20241107-spaceage";
    game-name = "Briefcase Full of Bees";
    #mods = depot.nix.pkgs.factorio-mods._all;
    #mods-dat = ./mod-settings.dat;
    admins = ["lukegb"];
    package = pkgs.factorio-headless-experimental;
    extraSettings = {
      auto_pause = true;
      only_admins_can_pause_the_game = false;
      game_password = depot.ops.secrets.factorioServerPassword;
      non_blocking_saving = true;
      autosave_only_on_server = true;
      autosave_interval = 5;
      autosave_slots = 60;
    };
  };

  my.hackyplayer = {
    enable = true;
    hostname = "hackyplayer.voc.emf.camp";
    workerConcurrency = 13;
  };
  my.emfminiserv = {
    enable = true;
    hostname = "prerelease.voc.emf.camp";
  };

  hardware.rasdaemon.enable = true;
  services.prometheus.exporters.ipmi = {
    enable = true;
  };

  my.services.seaweedfs = {
    securitySettings = {
      cors.allowed_origins.values = "*";
    };

    master = {
      enable = true;
      options = {
        port = 21000;
        ip = "rexxar.int.as205479.net";
        "ip.bind" = config.my.ip.tailscale6;
        mdir = "/var/lib/seaweedfs-master/metadata";
        defaultReplication = "000";
      };
    };
    filer = {
      enable = true;
      earlyOptions = {
        v = 4;
      };
      options = {
        port = 21010;
        ip = "rexxar.int.as205479.net";
        "ip.bind" = config.my.ip.tailscale6;
        master = "[${config.my.ip.tailscale6}]:21000";
        encryptVolumeData = true;

        s3 = true;
        "s3.domainName" = "objdump.zxcvbnm.ninja";
        "s3.localSocket" = "/run/seaweedfs-filer/s3.sock";
        "s3.port" = 21012;

        webdav = true;
        "webdav.port" = 21014;

        iam = true;
        "iam.ip" = "127.0.0.1";
        "iam.port" = 21015;

        defaultReplicaPlacement = "000";
      };
      settings = {
        leveldb2 = {
          enabled = true;
          dir = "/var/lib/seaweedfs-filer/leveldb2";
        };
      };
    };
    volume = {
      enable = true;
      earlyOptions = {
        v = 4;
      };
      options = {
        port = 21100;
        ip = "rexxar.int.as205479.net";
        "ip.bind" = config.my.ip.tailscale6;
        dataCenter = "ixn-lon1";
        rack = "vlx-lukegb1";
        max = 0;
        mserver = "[${config.my.ip.tailscale6}]:21000";
      };
    };
    cli.settings = {
      cluster.default = "rexxar";
      cluster.rexxar = {
        master = "[${config.my.ip.tailscale6}]:21000";
      };
    };
  };
  my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = {
    hostnames = [
      "objdump.zxcvbnm.ninja"
      "*.objdump.zxcvbnm.ninja"
    ];
    reloadOrRestartUnits = [ "caddy.service" ];
  };
  services.caddy = {
    enable = true;
    virtualHosts."objdump.zxcvbnm.ninja" = {
      serverAliases = [ "*.objdump.zxcvbnm.ninja" ];
      extraConfig = ''
        tls /var/lib/acme/objdump.zxcvbnm.ninja/fullchain.pem /var/lib/acme/objdump.zxcvbnm.ninja/privkey.pem
        reverse_proxy unix//run/seaweedfs-filer/s3.sock
      '';
    };
    virtualHosts."http://objdump.zxcvbnm.ninja" = {
      serverAliases = [ "http://*.objdump.zxcvbnm.ninja" ];
      extraConfig = ''
        reverse_proxy unix//run/seaweedfs-filer/s3.sock
      '';
    };
  };
  systemd.services.caddy.serviceConfig = {
    SupplementaryGroups = lib.mkAfter [ "acme" ];
    ReadOnlyPaths = lib.mkAfter [ "/var/lib/acme" ];
  };

  my.fup.listen = [];

  system.stateVersion = "24.05";
}