depot/ops/nixos/kusakabe/default.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, rebuilder, config, ... }:
let
  inherit (depot.ops) secrets;
  machineSecrets = secrets.machineSpecific.kusakabe;
in {
  imports = [
    ../lib/zfs.nix
  ];

  boot.initrd.availableKernelModules = [
    "ahci"
    "xhci_pci"
    "ehci_pci"
    "usbhid"
    "sd_mod"
  ];
  boot.kernelModules = [ "kvm-intel" ];

  nix.maxJobs = lib.mkDefault 8;

  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";

  fileSystems = let
    zfs = device: {
      device = device;
      fsType = "zfs";
    };
  in {
    "/" = zfs "tank/local/root";
    "/nix" = zfs "tank/local/nix";
    "/home" = zfs "tank/safe/home";
    "/store" = zfs "tank/local/store";
    "/persist" = zfs "tank/safe/persist";

    "/boot" = {
      device = "/dev/disk/by-label/EFIBOOT";
      fsType = "vfat";
    };
  };

  # Use systemd-boot.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  # Networking!
  networking = {
    hostName = "kusakabe";
    domain = "lukegb.com";
    hostId = "c6054536";

    nameservers = [
      "2001:4860:4860::8888"
      "2001:4860:4860::8844"
      "8.8.8.8"
      "8.8.4.4"
    ];
    useDHCP = false;
    defaultGateway = {
      address = "188.165.197.254";
      interface = "eno1";
    };
    defaultGateway6 = {
      address = "2001:41d0:2:8eff:ff:ff:ff:ff";
      interface = "eno1";
    };
    interfaces.eno1 = {
      ipv4.addresses = [{ address = "188.165.197.49"; prefixLength = 24; }];
      ipv6.addresses = [{ address = "2001:41d0:2:8e31::1"; prefixLength = 128; }];
      proxyARP = true;
    };
    interfaces.br-ext = {
      useDHCP = false;
      ipv4.addresses = [{ address = "137.74.77.17"; prefixLength = 28; }];
      ipv6.addresses = [{ address = "2001:41d0:2:8e31::2"; prefixLength = 64; }];
      proxyARP = true;
    };
    bridges.br-ext.interfaces = [];
    firewall = {
      interfaces.br-ext.allowedUDPPorts = [
        # DHCP
        67 68

        # TFTP
        69

        # NFS
        111 2049
      ];
      interfaces.br-ext.allowedTCPPorts = [
        # NFS
        111 2049

        # k8s
        80 443 6443 22623

        # PostgreSQL
        5432

        # XMPP
        5222 5223 5269 5347 5280 5281

        # TURN
        3478
      ];
      allowedUDPPorts = [
        # IPFS
        4001
      ];
      allowedTCPPorts = [
        80 443 6443
        5222 5223 5269 5280 5281
        3478

        # IPFS
        4001

        # rsyncd
        873
      ];
    };
  };
  my.ip.tailscale = "100.101.38.52";
  boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
  boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = 1;
  boot.kernel.sysctl."net.ipv6.conf.br-ext.proxy_ndp" = 1;
  boot.kernel.sysctl."net.ipv6.conf.eno1.proxy_ndp" = 1;

  services.dhcpd4 = {
    enable = true;
    interfaces = ["br-ext"];
    authoritative = true;
    extraConfig = ''
      option arch code 93 = unsigned integer 16;
      subnet 137.74.77.16 netmask 255.255.255.240 {
        option subnet-mask 255.255.255.240;
        option routers 137.74.77.17;
        option domain-name-servers 8.8.8.8, 8.8.4.4;
        option domain-name "kusakabe.lukegb.tech";
        deny unknown-clients;

        class "pxeclients" {
          match if substring (option vendor-class-identifier, 0, 9) = "PXEClient";
          next-server 137.74.77.17;
        }
        class "httpclients" {
          match if substring (option vendor-class-identifier, 0, 10) = "HTTPClient";
          option vendor-class-identifier "HTTPClient";
        }
        if exists user-class and option user-class = "iPXE" {
          filename "http://fed.lukegb.tech/ipxe.ipxe";
        } else if option arch = 00:07 {
          # x86-64 UEFI
          filename "netboot.xyz-snponly.efi";
        } else {
          # Legacy PXE
          filename "netboot.xyz-undionly.kpxe";
        }
      }
    '';
    machines = [{
      hostName = "fed";
      ethernetAddress = "02:00:00:d1:92:7a";
      ipAddress = "137.74.77.18";
    } {
      hostName = "okdboot";
      ethernetAddress = "52:54:00:52:40:96";
      ipAddress = "137.74.77.20";
    } {
      hostName = "okd1";
      ethernetAddress = "52:54:00:be:f4:2f";
      ipAddress = "137.74.77.21";
    } {
      hostName = "okd2";
      ethernetAddress = "52:54:00:92:86:09";
      ipAddress = "137.74.77.22";
    } {
      hostName = "okd3";
      ethernetAddress = "52:54:00:64:3d:3b";
      ipAddress = "137.74.77.23";
    }];
  };
  services.radvd = {
    enable = true;
    config = ''
      interface br-ext {
        AdvSendAdvert on;
        prefix 2001:41d0:2:8e31::/64 { };
        RDNSS 2001:4860:4860::8888 2001:4860:4860::8844 { };
      };
    '';
  };
  services.ndppd = {
    enable = true;
    proxies.eno1.rules."2001:41d0:2:8e31::/64" = {
      method = "static";
    };
  };
  services.atftpd = {
    enable = true;
    extraOptions = [
      "--bind-address=137.74.77.17"
    ];
  };

  # NFS
  services.nfs.server = {
    enable = true;
    exports = ''
      /export 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,wdelay,fsid=0,insecure,crossmnt)
      /export/openshift 137.74.77.16/28(rw,sync,nohide,no_subtree_check,no_root_squash,no_wdelay,insecure,crossmnt)
    '';
  };

  # LB
  services.haproxy = {
    enable = true;
    config = (let
      backends = { okd1 = "137.74.77.21"; okd2 = "137.74.77.22"; okd3 = "137.74.77.23"; };
      services = {
        k8sapi = { port = 6443; backendPort = 6443; sendProxy = false; backends = backends; };
        machineconfig = { port = 22623; backendPort = 22623; sendProxy = false; backends = backends; };
        https = { port = 443; backendPort = 443; sendProxy = true; backends = backends; };
        http = { port = 80; backendPort = 80; sendProxy = true; backends = backends; };
        xmpp-c2s = { port = 5222; backendPort = 32732; sendProxy = false; backends = backends; };
        xmpp-c2s-legacyssl = { port = 5223; backendPort = 31778; sendProxy = false; backends = backends; };
        xmpp-s2s = { port = 5269; backendPort = 32131; sendProxy = false; backends = backends; };
        xmpp-extcomp = { port = 5347; backendPort = 31856; sendProxy = false; backends = backends; };
        xmpp-http = { port = 5280; backendPort = 30389; sendProxy = false; backends = backends; };
        xmpp-https = { port = 5281; backendPort = 30952; sendProxy = false; backends = backends; };
      };
      backendToLine = backendPort: suffix: backendName: backendAddr: "server ${backendName} ${backendAddr}:${toString backendPort} ${suffix}";
      backendsToLine = backendPort: suffix: backends: lib.mapAttrsToList (backendToLine backendPort suffix) backends;
      serviceToFragment = serviceName: service: ''
        frontend ${serviceName}
          bind 137.74.77.17:${toString service.port}
          default_backend ${serviceName}-backend

        backend ${serviceName}-backend
          balance roundrobin
          ${lib.concatStringsSep "\n  " (backendsToLine service.backendPort "check ${if (service.sendProxy && false) then "send-proxy-v2" else ""}" service.backends)}
      '';
    in ''
      global
        maxconn 50000
        nbthread 4

      defaults
        log global
        mode tcp
        option tcplog
        maxconn 3000
        timeout connect 10s
        timeout client 1m
        timeout server 1m
        timeout tunnel 24h
        timeout client-fin 30s

      ${lib.concatStringsSep "\n\n" (lib.mapAttrsToList serviceToFragment services)}
    '');
  };

  virtualisation.libvirtd = {
    enable = true;
    qemuRunAsRoot = false;
    allowedBridges = [ "br-ext" ];
  };
  users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];

  services.postgresql = {
    enable = true;
    authentication = ''
      host all all 137.74.77.21/32 md5
      host all all 137.74.77.22/32 md5
      host all all 137.74.77.23/32 md5
    '';
    ensureDatabases = [
      "twitterchiver"
    ];
    ensureUsers = [{
      name = "twitterchiver";
      ensurePermissions = { "DATABASE twitterchiver" = "ALL PRIVILEGES"; };
    }];
    enableTCPIP = true;
  };
  services.postgresqlBackup.enable = true;

  services.coturn = {
    enable = true;
    use-auth-secret = true;
    realm = "turn.lukegb.com";
    static-auth-secret = machineSecrets.turnSecret;
  };

  services.ipfs = {
    enable = true;
    extraConfig = {
      Discovery.MDNS.Enabled = false;
      Swarm.DisableNatPortMap = true;
      Experimental.FilestoreEnabled = true;
      Addresses = {
        Swarm = [
          "/ip4/188.165.197.49/tcp/4001"
          "/ip6/2001:41d0:2:8e31::1/tcp/4001"
          "/ip4/188.165.197.49/udp/4001/quic"
          "/ip6/2001:41d0:2:8e31::1/udp/4001/quic"
        ];
      };
    };
    dataDir = "/store/ipfs";
  };

  services.rsyncd = {
    enable = true;
    settings = {
      global = {
        "use chroot" = false;
        "lock file" = "/run/rsync/rsyncd.lock";
        "max connections" = 4;
      };
      winworldpc = {
        path = "/store/winworldpc";
        comment = "WinWorldPC archive";
        "read only" = true;
        exclude = "/_data /_data/**";
      };
    };
  };
  users.users.rsync = {};
  systemd.services.rsyncd = {
    confinement.enable = true;
    serviceConfig = {
      User = "rsync";
      AmbientCapabilities = "CAP_NET_BIND_SERVICE";
      PrivateUsers = false;

      ProtectHostname = true;
      ProtectClock = true;
      RestrictAddressFamilies = "AF_INET AF_INET6";
      RestrictNamespaces = true;
      LockPersonality = true;
      NoNewPrivileges = true;
      MemoryDenyWriteExecute = true;

      RuntimeDirectory = "rsync";

      BindReadOnlyPaths = [
        "/store/winworldpc"
      ];
      BindPaths = [
        "/dev/log"
      ];
    };
  };

  system.stateVersion = "20.03";
}