depot/ops/nixos/lib/blade.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, rebuilder, config, ... }:
let
  inherit (depot.ops) secrets;
in {
  imports = [
    ../lib/zfs.nix
  ];

  options.my.blade = {
    bay = lib.mkOption {
      type = lib.types.int;
    };
    macAddress.internal = lib.mkOption {
      type = lib.types.str;
    };
    macAddress.storage = lib.mkOption {
      type = lib.types.str;
    };
    macAddress.internet = lib.mkOption {
      type = lib.types.nullOr lib.types.str;
      default = null;
    };
  };

  config = {
    boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
    boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];
    boot.kernelParams = [ "mitigations=off" ];
  
    fileSystems = let
      zfs = device: {
        device = device;
        fsType = "zfs";
      };
    in {
      "/" = zfs "tank/local/root";
      "/tmp" = zfs "tank/local/tmp";
      "/nix" = zfs "tank/local/nix";
      "/var" = zfs "tank/safe/var";
      "/home" = zfs "tank/safe/home";
      "/boot" = {
        device = "/dev/disk/by-label/boot";
        fsType = "ext4";
      };
    };
  
    boot.loader.grub.enable = true;
    boot.loader.grub.version = 2;
  
    # Networking!
    networking = {
      domain = "blade.as205479.net";
      nameservers = ["8.8.8.8" "8.8.4.4"];
      useDHCP = false;
      bridges = let
        br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
      in {
        br-mgmt = br [ "en-int" ];
        br-public = br [ "vl-int-public" ];
      };
      vlans.vl-int-public = {
        id = 100;
        interface = "en-int";
      };

      interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
        address = "10.100.0.${toString (100 + config.my.blade.bay)}";
        prefixLength = 23;
      }];
      interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
        address = "10.100.2.${toString (100 + config.my.blade.bay)}";
        prefixLength = 24;
      }];
  
      defaultGateway = lib.mkDefault "10.100.0.1";
  
      firewall.allowedUDPPorts = [
        41641 # Tailscale
      ];
      firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
      firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];

      nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) {
        enable = true;
        internalInterfaces = [ "br-mgmt" ];
        externalInterface = "en-internet";
      };
    };
    services.udev.extraRules = ''
      ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
      ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
    '' + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
      ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
    '');
  
    virtualisation.podman.enable = true;
  
    environment.systemPackages = with pkgs; [
      ceph
      xfsprogs
    ];
  
    services.ceph = {
      enable = true;
      global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
      global.monHost = "10.100.1.103, 10.100.1.106, 10.100.1.102";
      global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
      mon.daemons = [ config.networking.hostName ];
      mds.daemons = [ config.networking.hostName ];
      rgw.daemons = [ config.networking.hostName ];
      mgr.daemons = [ config.networking.hostName ];
      mgr.enable = config.services.ceph.mon.enable;
      rgw.enable = true;
    };
    systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
      enable = true;
      description = "Ceph OSD pre-start";
      before = [ "network-online.target" "ceph-osd.target" ];
      wantedBy = [ "ceph-osd.target" ];
  
      path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
  
      serviceConfig = {
        Type = "oneshot";
        ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
      };
    };
  
    virtualisation.libvirtd = {
      enable = true;
      qemuRunAsRoot = false;
      qemuPackage = pkgs.qemu_full;
      package = pkgs.libvirt.override {
        enableCeph = true;
        enableIscsi = true;
      };
    };
    security.polkit.enable = true;
    users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];

    services.journald.extraConfig = ''
      Storage=volatile
    '';
  
    system.stateVersion = "21.05";
  };
}