235 lines
7.4 KiB
Nix
235 lines
7.4 KiB
Nix
# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
{ depot, lib, pkgs, rebuilder, config, ... }:
|
|
let
|
|
inherit (depot.ops) secrets;
|
|
in {
|
|
imports = [
|
|
../lib/zfs.nix
|
|
];
|
|
|
|
options.my.blade = {
|
|
bay = lib.mkOption {
|
|
type = lib.types.int;
|
|
};
|
|
macAddress.internal = lib.mkOption {
|
|
type = lib.types.str;
|
|
};
|
|
macAddress.storage = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
};
|
|
macAddress.internet = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
};
|
|
macAddress.public = lib.mkOption {
|
|
# If not using a VLAN.
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
};
|
|
};
|
|
|
|
config = {
|
|
boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
|
|
boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];
|
|
|
|
# Enable serial console.
|
|
boot.loader.grub.extraConfig = ''
|
|
serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1
|
|
terminal_input console serial
|
|
terminal_output console serial
|
|
'';
|
|
boot.kernelParams = [
|
|
"console=tty1"
|
|
"console=ttyS0,115200" # <-- /dev/console
|
|
|
|
"mitigations=off"
|
|
];
|
|
|
|
my.rundeck.tags = [ "blade" ];
|
|
my.home-manager.imports = lib.mkAfter [ ./home-manager/blade.nix ];
|
|
|
|
fileSystems = let
|
|
zfs = device: {
|
|
device = device;
|
|
fsType = "zfs";
|
|
};
|
|
tmpfs = size: {
|
|
device = "none";
|
|
fsType = "tmpfs";
|
|
options = [ "defaults" "size=${size}" "mode=755" ];
|
|
};
|
|
in {
|
|
"/" = zfs "tank/local/root";
|
|
"/tmp" = zfs "tank/local/tmp";
|
|
"/nix" = zfs "tank/local/nix";
|
|
"/var" = zfs "tank/safe/var";
|
|
"/home" = zfs "tank/safe/home";
|
|
"/boot" = {
|
|
device = "/dev/disk/by-label/boot";
|
|
fsType = "ext4";
|
|
};
|
|
"/var/log" = tmpfs "2G";
|
|
"/var/cache" = tmpfs "16G";
|
|
} // (lib.optionalAttrs (config.services.ceph.osd.enable || config.services.ceph.mgr.enable || config.services.ceph.mon.enable || config.services.ceph.mgr.enable) {
|
|
"/var/lib/ceph" = {
|
|
device = "/dev/disk/by-label/var-lib-ceph";
|
|
fsType = "xfs";
|
|
};
|
|
});
|
|
|
|
boot.loader.grub.enable = true;
|
|
boot.loader.grub.version = 2;
|
|
|
|
# Networking!
|
|
networking = {
|
|
domain = "blade.as205479.net";
|
|
nameservers = ["8.8.8.8" "8.8.4.4"];
|
|
search = lib.mkBefore [
|
|
"blade.as205479.net"
|
|
"storage.blade.as205479.net"
|
|
];
|
|
bridges = let
|
|
br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
|
|
in {
|
|
br-mgmt = br [ "en-int" ];
|
|
br-public = br [ (if config.my.blade.macAddress.public == null then "vl-int-public" else "en-public") ];
|
|
};
|
|
vlans = ({} //
|
|
(if config.my.blade.macAddress.public == null then {
|
|
vl-int-public = {
|
|
id = 100;
|
|
interface = "en-int";
|
|
};
|
|
} else {}));
|
|
|
|
interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
|
|
address = "10.100.0.${toString (100 + config.my.blade.bay)}";
|
|
prefixLength = 23;
|
|
}];
|
|
interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
|
|
address = "10.100.2.${toString (100 + config.my.blade.bay)}";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
defaultGateway = lib.mkDefault "10.100.0.1";
|
|
|
|
firewall.allowedUDPPorts = [
|
|
41641 # Tailscale
|
|
];
|
|
firewall.interfaces.en-storage.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
|
|
firewall.interfaces.en-storage.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];
|
|
|
|
firewall.extraCommands = ''
|
|
iptables -A nixos-fw -i en-storage -s 10.100.2.0/23 -j ACCEPT
|
|
iptables -A nixos-fw -i br-mgmt -s 10.100.0.0/23 -j ACCEPT
|
|
'';
|
|
|
|
nat = lib.optionalAttrs (config.my.blade.macAddress.internet != null) {
|
|
enable = true;
|
|
internalInterfaces = [ "br-mgmt" ];
|
|
externalInterface = "vl-transit";
|
|
externalIP = "92.118.28.1";
|
|
};
|
|
};
|
|
services.udev.extraRules = ''
|
|
ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
|
|
'' + (lib.optionalString (config.my.blade.macAddress.storage != null) ''
|
|
ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
|
|
'') + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
|
|
ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
|
|
'') + (lib.optionalString (config.my.blade.macAddress.public != null) ''
|
|
ATTR{address}=="${config.my.blade.macAddress.public}", NAME="en-public"
|
|
'');
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
ceph
|
|
xfsprogs
|
|
];
|
|
|
|
services.ceph = {
|
|
enable = true;
|
|
global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
|
|
global.monHost = "10.100.2.103, 10.100.2.106, 10.100.2.102";
|
|
global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
|
|
global.publicNetwork = "10.100.2.0/24";
|
|
global.clusterNetwork = "10.100.2.0/24";
|
|
extraConfig.rgw_dns_name = "objdump.zxcvbnm.ninja";
|
|
extraConfig.rgw_data_log_backing = "omap";
|
|
extraConfig.rgw_default_data_log_backing = "omap";
|
|
mon.daemons = [ config.networking.hostName ];
|
|
mds.daemons = [ config.networking.hostName ];
|
|
rgw.daemons = [ config.networking.hostName ];
|
|
mgr.daemons = [ config.networking.hostName ];
|
|
mgr.enable = config.services.ceph.mon.enable;
|
|
rgw.enable = true;
|
|
client.enable = true;
|
|
client.extraConfig = {
|
|
"client.libvirt" = {
|
|
rbd_cache = "true";
|
|
rbd_cache_policy = "writeback";
|
|
rbd_cache_size = "2Gi";
|
|
rbd_cache_max_dirty = "1792Mi";
|
|
rbd_cache_target_dirty = "128Mi";
|
|
};
|
|
};
|
|
};
|
|
systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
|
|
enable = true;
|
|
description = "Ceph OSD pre-start";
|
|
before = [ "network-online.target" "ceph-osd.target" ];
|
|
wantedBy = [ "ceph-osd.target" ];
|
|
|
|
path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
|
|
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
|
|
};
|
|
};
|
|
|
|
virtualisation.libvirtd = {
|
|
enable = true;
|
|
qemu = {
|
|
runAsRoot = true;
|
|
package = pkgs.qemu.override {
|
|
gtkSupport = false;
|
|
sdlSupport = false;
|
|
spiceSupport = true;
|
|
cephSupport = true;
|
|
smartcardSupport = false;
|
|
pulseSupport = false;
|
|
alsaSupport = false;
|
|
libiscsiSupport = false;
|
|
|
|
hostCpuOnly = true;
|
|
};
|
|
};
|
|
package = pkgs.libvirt.override {
|
|
enableCeph = true;
|
|
enableIscsi = true;
|
|
};
|
|
};
|
|
security.polkit.enable = true;
|
|
users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];
|
|
|
|
# Our disk is slow; don't write to it...
|
|
services.journald.extraConfig = ''
|
|
Storage=volatile
|
|
'';
|
|
systemd.coredump.extraConfig = ''
|
|
Storage=none
|
|
ProcessSizeMax=0
|
|
'';
|
|
|
|
fonts.fontconfig.enable = false;
|
|
programs.ssh.setXAuthLocation = false;
|
|
security.pam.services.su.forwardXAuth = lib.mkForce false;
|
|
services.fwupd.enable = lib.mkForce false;
|
|
xdg.sounds.enable = false;
|
|
|
|
system.stateVersion = "21.05";
|
|
};
|
|
}
|