depot/ops/nixos/lib/blade.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, rebuilder, config, ... }:
let
  inherit (depot.ops) secrets;
in {
  imports = [
    ../lib/minimal.nix
    ../lib/zfs.nix
  ];

  options.my.blade = {
    bay = lib.mkOption {
      type = lib.types.int;
    };
    macAddress.internal = lib.mkOption {
      type = lib.types.str;
    };
    macAddress.storage = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
    };
    macAddress.internet = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
    };
    macAddress.public = lib.mkOption {
      # If not using a VLAN.
      type = lib.types.nullOr lib.types.str;
      default = null;
    };

    onZFS = lib.mkOption { type = lib.types.bool; default = true; };
  };

  config = {
    boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
    boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];

    # Enable serial console.
    boot.loader.grub.extraConfig = ''
      serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
      terminal_input console serial
      terminal_output console serial
    '';
    boot.kernelParams = [
      "console=tty1"
      "console=ttyS0,115200"  # <-- /dev/console

      "mitigations=off"
    ];

    my.rundeck.tags = [ "blade" ];

    fileSystems = let
      zfs = device: {
        device = device;
        fsType = "zfs";
      };
      tmpfs = size: {
        device = "none";
        fsType = "tmpfs";
        options = [ "defaults" "size=${size}" "mode=755" ];
      };
    in {
      "/boot" = {
        device = "/dev/disk/by-label/boot";
        fsType = "ext4";
      };
      "/var/log" = tmpfs "2G";
      "/var/cache" = tmpfs "16G";
    } // (lib.optionalAttrs (config.my.blade.onZFS) {
      "/" = zfs "tank/local/root";
      "/tmp" = zfs "tank/local/tmp";
      "/nix" = zfs "tank/local/nix";
      "/var" = zfs "tank/safe/var";
      "/home" = zfs "tank/safe/home";
    }) // (lib.optionalAttrs (!config.my.blade.onZFS) {
      "/" = {
        device = "/dev/disk/by-label/root";
        fsType = "ext4";
      };
    }) // (lib.optionalAttrs (config.services.ceph.osd.enable || config.services.ceph.mgr.enable || config.services.ceph.mon.enable || config.services.ceph.mgr.enable) {
      "/var/lib/ceph" = {
        device = "/dev/disk/by-label/var-lib-ceph";
        fsType = "xfs";
      };
    });

    boot.loader.grub.enable = true;

    # Networking!
    networking = {
      domain = "blade.as205479.net";
      nameservers = ["8.8.8.8" "8.8.4.4"];
      search = lib.mkBefore [
        "blade.as205479.net"
        "storage.blade.as205479.net"
      ];
      bridges = let
        br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
      in {
        br-mgmt = br [ "en-int" ];
        br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ];
      };
      vlans = ({} //
        (if config.my.blade.macAddress.public == null then {
          vl-int-public = {
            id = 100;
            interface = "en-int";
          };
        } else {}));

      interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
        address = "10.100.0.${toString (100 + config.my.blade.bay)}";
        prefixLength = 23;
      }];
      interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
        address = "10.100.2.${toString (100 + config.my.blade.bay)}";
        prefixLength = 24;
      }];

      defaultGateway = lib.mkDefault "10.100.0.1";

      firewall.allowedUDPPorts = [
        41641 # Tailscale
      ];
      firewall.interfaces.en-storage.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
      firewall.interfaces.en-storage.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];

      firewall.extraCommands = ''
        iptables -A nixos-fw -i en-storage -s 10.100.2.0/23 -j ACCEPT
        iptables -A nixos-fw -i br-mgmt -s 10.100.0.0/23 -j ACCEPT
      '';

      nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) {
        enable = true;
        internalInterfaces = [ "br-mgmt" ];
        externalInterface = "vl-transit";
        externalIP = "92.118.28.1";
      };
    };
    services.udev.extraRules = ''
      ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
    '' + (lib.optionalString (config.my.blade.macAddress.storage != null) ''
      ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
    '') + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
      ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
    '') + (lib.optionalString (config.my.blade.macAddress.public != null) ''
      ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public"
    '');

    environment.systemPackages = with pkgs; [
      ceph
      xfsprogs
    ];

    services.ceph = {
      enable = true;
      global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
      global.monHost = "10.100.2.103, 10.100.2.106, 10.100.2.102";
      global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
      global.publicNetwork = "10.100.2.0/24";
      global.clusterNetwork = "10.100.2.0/24";
      extraConfig.rgw_dns_name = "objdump.zxcvbnm.ninja";
      extraConfig.rgw_data_log_backing = "omap";
      extraConfig.rgw_default_data_log_backing = "omap";
      mon.daemons = [ config.networking.hostName ];
      mds.daemons = [ config.networking.hostName ];
      rgw.daemons = [ config.networking.hostName ];
      mgr.daemons = [ config.networking.hostName ];
      mgr.enable = config.services.ceph.mon.enable;
      rgw.enable = true;
      client.enable = true;
      client.extraConfig = {
        "client.libvirt" = {
          rbd_cache = "true";
          rbd_cache_policy = "writeback";
          rbd_cache_size = "2Gi";
          rbd_cache_max_dirty = "1792Mi";
          rbd_cache_target_dirty = "128Mi";
        };
      };
    };
    systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
      enable = true;
      description = "Ceph OSD pre-start";
      before = [ "network-online.target" "ceph-osd.target" ];
      wantedBy = [ "ceph-osd.target" ];

      path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];

      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
      };
    };

    virtualisation.libvirtd = {
      enable = true;
      qemu = {
        runAsRoot = true;
        package = pkgs.qemu.override {
          gtkSupport = false;
          sdlSupport = false;
          spiceSupport = true;
          cephSupport = true;
          smartcardSupport = false;
          pulseSupport = false;
          alsaSupport = false;
          libiscsiSupport = false;

          hostCpuOnly = true;
        };
      };
      package = pkgs.libvirt.override {
        enableCeph = true;
        enableIscsi = true;
      };
    };
    security.polkit.enable = true;
    users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];

    # Our disk is slow; don't write to it...
    services.journald.extraConfig = ''
      Storage=volatile
    '';
    systemd.coredump.extraConfig = ''
      Storage=none
      ProcessSizeMax=0
    '';

    system.stateVersion = "21.05";
  };
}