depot/ops/nixos/cofractal-ams01/default.nix

# SPDX-FileCopyrightText: 2023 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, config, ... }:
{
  imports = [
    ../lib/zfs.nix
    ../lib/bgp.nix
    ../lib/nixbuild-distributed.nix
    ../lib/gitlab-runner-cacher.nix
  ];

  # Otherwise _this_ machine won't enumerate things properly.
  boot.zfs.devNodes = "/dev/disk/by-id";

  boot.initrd = {
    availableKernelModules = [
      "nvme"
      "xhci_pci"
      "ahci"
      "usb_storage"
      "usbhid"
      "sd_mod"
      "sr_mod"
    ];
  };
  boot.kernelModules = [ "kvm-amd" ];
  hardware.cpu.amd.updateMicrocode = true;

  # Use the systemd-boot EFI boot loader.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  boot.blacklistedKernelModules = [ "ib_core" "irdma" ];

  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";
  services.zfs.rollbackOnBoot = {
    enable = true;
    snapshot = "zfast/local/root@blank";
  };

  fileSystems = let
    zfs = device: {
      device = device;
      fsType = "zfs";
    };
  in {
    "/" = zfs "zfast/local/root";
    "/nix" = zfs "zfast/local/nix";
    "/tmp" = zfs "zfast/local/tmp";

    "/persist" = zfs "zfast/safe/persist";
    "/home" = (zfs "zslow/safe/home") // { neededForBoot = true; };

    "/boot" = {
      device = "/dev/disk/by-label/ESP";
      fsType = "vfat";
    };
    "/boot2" = {
      device = "/dev/disk/by-label/ESP2";
      fsType = "vfat";
    };
  };
  boot.loader.systemd-boot.extraInstallCommands = ''
    rsync -a /boot/ /boot2/
  '';

  nix.settings.max-jobs = lib.mkDefault 8;

  # Networking!
  networking = {
    hostName = "cofractal-ams01";
    domain = "as205479.net";
    hostId = "a1cf1a9f";
    useNetworkd = true;

    nameservers = [
      "2001:4860:4860::8888"
      "2001:4860:4860::8844"
      "8.8.8.8"
      "8.8.4.4"
    ];
    bonds.bond0 = {
      interfaces = [ "enp45s0f0" "enp45s0f1" ];
      driverOptions = {
        miimon = "100";
        mode = "802.3ad";
      };
    };
    defaultGateway6.address = "2a09:a446:1337:ffff::1";
    interfaces.bond0 = {
      ipv6.addresses = [
        { address = "2a09:a446:1337::10"; prefixLength = 64; }
        { address = "2a09:a446:1337:ffff::10"; prefixLength = 120; }
      ];
      ipv4.addresses = [
        { address = "199.19.152.160"; prefixLength = 30; }
      ];
    };
  };
  my.ip.tailscale = "100.94.187.27";
  my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:625e:bb1b";

  services.openssh.hostKeys = [
    {
      path = "/persist/etc/ssh/ssh_host_ed25519_key";
      type = "ed25519";
    }
    {
      path = "/persist/etc/ssh/ssh_host_rsa_key";
      type = "rsa";
      bits = 4096;
    }
  ];

  systemd.mounts = let
    bindMount' = dir: {
      unitConfig.RequiresMountsFor = dir;
      options = "bind";
      what = "/persist${dir}";
      where = dir;
    };
    bindMountSvc = dir: svc: (bindMount' dir) // {
      bindsTo = [svc];
      partOf = [svc];
    };
    bindMountSvcDynamic = dir: svc: (bindMount' "/var/lib/private/${dir}") // {
      requiredBy = [svc];
      before = [svc];
      wantedBy = ["multi-user.target"];
    };
    bindMount = dir: (bindMount' dir) // {
      wantedBy = ["multi-user.target"];
    };
  in [
    (bindMountSvc "/var/lib/tailscale" "tailscaled.service")
  ];

  services.lukegbgp = let
    local.asn = 205479;
  in {
    enable = true;
    config = {
      local = {
        routerID = "199.19.152.160";
      };
      export.v4 = [ ];
      export.v6 = [ "2a09:a446:1337::/48" ];
      peering.cofractal = {
        local = local // {
          v6 = "2a09:a446:1337:ffff::10";
        };
        v4onv6 = true;
        remote = {
          asn = 26073;
          export_community = 6000;
          routers = [{
            v6 = "2a09:a446:1337:ffff::2";
          } {
            v6 = "2a09:a446:1337:ffff::3";
          }];
        };
      };
    };
  };

  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];

  system.stateVersion = "23.05";
}