# SPDX-FileCopyrightText: 2020 Luke Granger-Brown # # SPDX-License-Identifier: Apache-2.0 { depot, lib, pkgs, rebuilder, config, ... }: let inherit (depot.ops) secrets; in { imports = [ ../lib/minimal.nix ../lib/zfs.nix ]; options.my.blade = { bay = lib.mkOption { type = lib.types.int; }; macAddress.internal = lib.mkOption { type = lib.types.str; }; macAddress.storage = lib.mkOption { type = lib.types.nullOr lib.types.str; }; macAddress.internet = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; }; macAddress.public = lib.mkOption { # If not using a VLAN. type = lib.types.nullOr lib.types.str; default = null; }; }; config = { boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ]; # Enable serial console. boot.loader.grub.extraConfig = '' serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1 terminal_input console serial terminal_output console serial ''; boot.kernelParams = [ "console=tty1" "console=ttyS0,115200" # <-- /dev/console "mitigations=off" ]; my.rundeck.tags = [ "blade" ]; fileSystems = let zfs = device: { device = device; fsType = "zfs"; }; tmpfs = size: { device = "none"; fsType = "tmpfs"; options = [ "defaults" "size=${size}" "mode=755" ]; }; in { "/" = zfs "tank/local/root"; "/tmp" = zfs "tank/local/tmp"; "/nix" = zfs "tank/local/nix"; "/var" = zfs "tank/safe/var"; "/home" = zfs "tank/safe/home"; "/boot" = { device = "/dev/disk/by-label/boot"; fsType = "ext4"; }; "/var/log" = tmpfs "2G"; "/var/cache" = tmpfs "16G"; } // (lib.optionalAttrs (config.services.ceph.osd.enable || config.services.ceph.mgr.enable || config.services.ceph.mon.enable || config.services.ceph.mgr.enable) { "/var/lib/ceph" = { device = "/dev/disk/by-label/var-lib-ceph"; fsType = "xfs"; }; }); boot.loader.grub.enable = true; boot.loader.grub.version = 2; # Networking! networking = { domain = "blade.as205479.net"; nameservers = ["8.8.8.8" "8.8.4.4"]; search = lib.mkBefore [ "blade.as205479.net" "storage.blade.as205479.net" ]; bridges = let br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; }; in { br-mgmt = br [ "en-int" ]; br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ]; }; vlans = ({} // (if config.my.blade.macAddress.public == null then { vl-int-public = { id = 100; interface = "en-int"; }; } else {})); interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{ address = "10.100.0.${toString (100 + config.my.blade.bay)}"; prefixLength = 23; }]; interfaces.en-storage.ipv4.addresses = lib.mkBefore [{ address = "10.100.2.${toString (100 + config.my.blade.bay)}"; prefixLength = 24; }]; defaultGateway = lib.mkDefault "10.100.0.1"; firewall.allowedUDPPorts = [ 41641 # Tailscale ]; firewall.interfaces.en-storage.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ]; firewall.interfaces.en-storage.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }]; firewall.extraCommands = '' iptables -A nixos-fw -i en-storage -s 10.100.2.0/23 -j ACCEPT iptables -A nixos-fw -i br-mgmt -s 10.100.0.0/23 -j ACCEPT ''; nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) { enable = true; internalInterfaces = [ "br-mgmt" ]; externalInterface = "vl-transit"; externalIP = "92.118.28.1"; }; }; services.udev.extraRules = '' ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int" '' + (lib.optionalString (config.my.blade.macAddress.storage != null) '' ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage" '') + (lib.optionalString (config.my.blade.macAddress.internet != null) '' ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet" '') + (lib.optionalString (config.my.blade.macAddress.public != null) '' ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public" ''); environment.systemPackages = with pkgs; [ ceph xfsprogs ]; services.ceph = { enable = true; global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; global.monHost = "10.100.2.103, 10.100.2.106, 10.100.2.102"; global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris"; global.publicNetwork = "10.100.2.0/24"; global.clusterNetwork = "10.100.2.0/24"; extraConfig.rgw_dns_name = "objdump.zxcvbnm.ninja"; extraConfig.rgw_data_log_backing = "omap"; extraConfig.rgw_default_data_log_backing = "omap"; mon.daemons = [ config.networking.hostName ]; mds.daemons = [ config.networking.hostName ]; rgw.daemons = [ config.networking.hostName ]; mgr.daemons = [ config.networking.hostName ]; mgr.enable = config.services.ceph.mon.enable; rgw.enable = true; client.enable = true; client.extraConfig = { "client.libvirt" = { rbd_cache = "true"; rbd_cache_policy = "writeback"; rbd_cache_size = "2Gi"; rbd_cache_max_dirty = "1792Mi"; rbd_cache_target_dirty = "128Mi"; }; }; }; systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable { enable = true; description = "Ceph OSD pre-start"; before = [ "network-online.target" "ceph-osd.target" ]; wantedBy = [ "ceph-osd.target" ]; path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; serviceConfig = { Type = "oneshot"; ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; }; }; virtualisation.libvirtd = { enable = true; qemu = { runAsRoot = true; package = pkgs.qemu.override { gtkSupport = false; sdlSupport = false; spiceSupport = true; cephSupport = true; smartcardSupport = false; pulseSupport = false; alsaSupport = false; libiscsiSupport = false; hostCpuOnly = true; }; }; package = pkgs.libvirt.override { enableCeph = true; enableIscsi = true; }; }; security.polkit.enable = true; users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ]; # Our disk is slow; don't write to it... services.journald.extraConfig = '' Storage=volatile ''; systemd.coredump.extraConfig = '' Storage=none ProcessSizeMax=0 ''; system.stateVersion = "21.05"; }; }