depot/ops/nixos/kusakabe/default.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, rebuilder, config, ... }:
let
  inherit (depot.ops) secrets;
  machineSecrets = secrets.machineSpecific.kusakabe;
in {
  imports = [
    ../lib/zfs.nix
  ];

  boot.initrd.availableKernelModules = [
    "ahci"
    "xhci_pci"
    "ehci_pci"
    "usbhid"
    "sd_mod"
  ];
  boot.kernelModules = [ "kvm-intel" ];

  nix.maxJobs = lib.mkDefault 8;

  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";

  fileSystems = let
    zfs = device: {
      device = device;
      fsType = "zfs";
    };
  in {
    "/" = zfs "tank/local/root";
    "/nix" = zfs "tank/local/nix";
    "/home" = zfs "tank/safe/home";

    "/boot" = {
      device = "/dev/disk/by-label/EFIBOOT";
      fsType = "vfat";
    };
  };

  # Use systemd-boot.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  # Networking!
  networking = {
    hostName = "kusakabe";
    domain = "lukegb.com";
    hostId = "c6054536";

    nameservers = [
      "2001:4860:4860::8888"
      "2001:4860:4860::8844"
      "8.8.8.8"
      "8.8.4.4"
    ];
    useDHCP = false;
    defaultGateway = {
      address = "188.165.197.254";
      interface = "eno1";
    };
    defaultGateway6 = {
      address = "2001:41d0:2:8eff:ff:ff:ff:ff";
      interface = "eno1";
    };
    interfaces.eno1 = {
      ipv4.addresses = [{ address = "188.165.197.49"; prefixLength = 24; }];
      ipv6.addresses = [{ address = "2001:41d0:2:8e31::1"; prefixLength = 128; }];
      proxyARP = true;
    };
    interfaces.br-ext = {
      useDHCP = false;
      ipv4.addresses = [{ address = "137.74.77.17"; prefixLength = 28; }];
      ipv6.addresses = [{ address = "2001:41d0:2:8e31::2"; prefixLength = 64; }];
      proxyARP = true;
    };
    bridges.br-ext.interfaces = [];
    firewall = {
      interfaces.br-ext.allowedUDPPorts = [
        # DHCP
        67 68

        # TFTP
        69

        # NFS
        111 2049
      ];
      interfaces.br-ext.allowedTCPPorts = [
        # NFS
        111 2049

        # k8s
        80 443 6443 22623
      ];
      allowedTCPPorts = [ 80 443 ];
    };
  };
  boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
  boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
  boot.kernel.sysctl."net.ipv6.conf.br-ext.proxy_ndp" = 1;
  boot.kernel.sysctl."net.ipv6.conf.eno1.proxy_ndp" = 1;

  services.dhcpd4 = {
    enable = true;
    interfaces = ["br-ext"];
    authoritative = true;
    extraConfig = ''
      option arch code 93 = unsigned integer 16;
      subnet 137.74.77.16 netmask 255.255.255.240 {
        option subnet-mask 255.255.255.240;
        option routers 137.74.77.17;
        option domain-name-servers 8.8.8.8, 8.8.4.4;
        option domain-name "kusakabe.lukegb.tech";
        deny unknown-clients;

        class "pxeclients" {
          match if substring (option vendor-class-identifier, 0, 9) = "PXEClient";
          next-server 137.74.77.17;
        }
        class "httpclients" {
          match if substring (option vendor-class-identifier, 0, 10) = "HTTPClient";
          option vendor-class-identifier "HTTPClient";
        }
        if exists user-class and option user-class = "iPXE" {
          filename "http://fed.lukegb.tech/ipxe.ipxe";
        } else if option arch = 00:07 {
          # x86-64 UEFI
          filename "netboot.xyz-snponly.efi";
        } else {
          # Legacy PXE
          filename "netboot.xyz-undionly.kpxe";
        }
      }
    '';
    machines = [{
      hostName = "fed";
      ethernetAddress = "02:00:00:d1:92:7a";
      ipAddress = "137.74.77.18";
    } {
      hostName = "okdboot";
      ethernetAddress = "52:54:00:52:40:96";
      ipAddress = "137.74.77.20";
    } {
      hostName = "okd1";
      ethernetAddress = "52:54:00:be:f4:2f";
      ipAddress = "137.74.77.21";
    } {
      hostName = "okd2";
      ethernetAddress = "52:54:00:92:86:09";
      ipAddress = "137.74.77.22";
    } {
      hostName = "okd3";
      ethernetAddress = "52:54:00:64:3d:3b";
      ipAddress = "137.74.77.23";
    }];
  };
  services.radvd = {
    enable = true;
    config = ''
      interface br-ext {
        AdvSendAdvert on;
        prefix 2001:41d0:2:8e31::/64 { };
        RDNSS 2001:4860:4860::8888 2001:4860:4860::8844 { };
      };
    '';
  };
  services.ndppd = {
    enable = true;
    proxies.eno1.rules."2001:41d0:2:8e31::/64" = {
      method = "static";
    };
  };
  services.atftpd = {
    enable = true;
    extraOptions = [
      "--bind-address=137.74.77.17"
    ];
  };

  # NFS
  services.nfs.server = {
    enable = true;
    exports = ''
      /export 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,wdelay,fsid=0,insecure,crossmnt)
      /export/openshift 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,no_wdelay,insecure,crossmnt)
    '';
  };

  # LB
  services.haproxy = {
    enable = true;
    config = ''
      global
        maxconn 50000
        nbthread 4

      defaults
        log global
        mode tcp
        option tcplog
        maxconn 3000
        timeout connect 10s
        timeout client 1m
        timeout server 1m

      frontend k8sapi
        bind 137.74.77.17:6443
        default_backend k8sapi-backend

      backend k8sapi-backend
        balance roundrobin
        mode tcp
        server okd1 137.74.77.21:6443 check
        server okd2 137.74.77.22:6443 check
        server okd3 137.74.77.23:6443 check

      frontend machineconfig
        bind 137.74.77.17:22623
        default_backend machineconfig-backend

      backend machineconfig-backend
        balance roundrobin
        mode tcp
        server okd1 137.74.77.21:22623 check
        server okd2 137.74.77.22:22623 check
        server okd3 137.74.77.23:22623 check

      frontend https
        bind 137.74.77.17:443
        default_backend https-backend

      backend https-backend
        balance roundrobin
        mode tcp
        server okd1 137.74.77.21:443 check
        server okd2 137.74.77.22:443 check
        server okd3 137.74.77.23:443 check

      frontend http
        bind 137.74.77.17:80
        default_backend http-backend

      backend http-backend
        balance roundrobin
        mode tcp
        server okd1 137.74.77.21:80 check
        server okd2 137.74.77.22:80 check
        server okd3 137.74.77.23:80 check
    '';
  };

  virtualisation.libvirtd = {
    enable = true;
    qemuRunAsRoot = false;
    allowedBridges = [ "br-ext" ];
  };
  users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];

  system.stateVersion = "20.03";
}