From 53b7ca1c8a83a574bda1c99b923ce733ca165b9d Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Fri, 12 Mar 2021 14:47:08 +0000 Subject: [PATCH] ops/nixos: revamp blade network config --- ops/nixos/blade-chakotay/default.nix | 9 +- ops/nixos/blade-janeway/default.nix | 12 +- ops/nixos/blade-kim/default.nix | 9 +- ops/nixos/blade-paris/default.nix | 27 +++- ops/nixos/blade-torres/default.nix | 9 +- ops/nixos/blade-tuvok/default.nix | 18 ++- ops/nixos/lib/blade.nix | 194 ++++++++++++++++----------- 7 files changed, 175 insertions(+), 103 deletions(-) diff --git a/ops/nixos/blade-chakotay/default.nix b/ops/nixos/blade-chakotay/default.nix index 46b3522639..c4fad0958d 100644 --- a/ops/nixos/blade-chakotay/default.nix +++ b/ops/nixos/blade-chakotay/default.nix @@ -16,12 +16,13 @@ in { networking = { hostName = "blade-chakotay"; hostId = "40bc5a75"; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.182"; - prefixLength = 24; - }]; }; my.ip.tailscale = "100.121.11.7"; + my.blade.bay = 5; + my.blade.macAddress = { + internal = "e4:11:5b:ac:e3:b8"; + storage = "e4:11:5b:ac:e3:bc"; + }; services.ceph = { #osd.enable = true; diff --git a/ops/nixos/blade-janeway/default.nix b/ops/nixos/blade-janeway/default.nix index 61e204b4c7..df7cab725c 100644 --- a/ops/nixos/blade-janeway/default.nix +++ b/ops/nixos/blade-janeway/default.nix @@ -16,15 +16,13 @@ in { networking = { hostName = "blade-janeway"; hostId = "3a62390f"; - bridges.br-mgmt.interfaces = [ "enp4s0f0" ]; - bridges.br-storage.interfaces = [ "enp4s0f1" ]; - bridges.br-ext.interfaces = [ "enp5s0f0" ]; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.180"; - prefixLength = 24; - }]; }; my.ip.tailscale = "100.121.116.85"; + my.blade.bay = 3; + my.blade.macAddress = { + internal = "e8:39:35:1f:7f:8a"; + storage = "e8:39:35:1f:7f:8e"; + }; services.ceph = { mon.enable = true; diff --git a/ops/nixos/blade-kim/default.nix b/ops/nixos/blade-kim/default.nix index fa29ec8c40..ffc0cc5515 100644 --- a/ops/nixos/blade-kim/default.nix +++ b/ops/nixos/blade-kim/default.nix @@ -16,12 +16,13 @@ in { networking = { hostName = "blade-kim"; hostId = "1643efb6"; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.183"; - prefixLength = 24; - }]; }; my.ip.tailscale = "100.84.36.62"; + my.blade.bay = 1; + my.blade.macAddress = { + internal = "e4:11:5b:ac:d1:7a"; + storage = "e4:11:5b:ac:d1:7e"; + }; services.ceph = { #osd.enable = true; diff --git a/ops/nixos/blade-paris/default.nix b/ops/nixos/blade-paris/default.nix index 223da918ef..0028c56e7c 100644 --- a/ops/nixos/blade-paris/default.nix +++ b/ops/nixos/blade-paris/default.nix @@ -16,12 +16,35 @@ in { networking = { hostName = "blade-paris"; hostId = "41b2a198"; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.184"; + interfaces.br-mgmt.ipv4.addresses = [{ + address = "10.100.0.1"; + prefixLength = 23; + }]; + interfaces.br-public.ipv4.addresses = [{ + address = "92.118.28.1"; prefixLength = 24; }]; + interfaces.en-internet.ipv4.addresses = [{ + address = "192.168.1.184"; + prefixLength = 24; + } { + address = "195.74.55.23"; + prefixLength = 31; + }]; + interfaces.en-internet.ipv6.addresses = [{ + address = "2a03:ee40:8080:9:2::2"; + prefixLength = 126; + }]; + defaultGateway = "192.168.1.5"; + defaultGateway6 = "2a03:ee40:8080:9:2::1"; }; my.ip.tailscale = "100.117.185.118"; + my.blade.bay = 2; + my.blade.macAddress = { + internal = "e4:11:5b:ac:e4:8a"; + storage = "e4:11:5b:ac:e4:8e"; + internet = "e4:11:5b:ac:e4:8c"; + }; services.ceph = { mon.enable = true; diff --git a/ops/nixos/blade-torres/default.nix b/ops/nixos/blade-torres/default.nix index 9c7ff62c3f..788f997d15 100644 --- a/ops/nixos/blade-torres/default.nix +++ b/ops/nixos/blade-torres/default.nix @@ -16,12 +16,13 @@ in { networking = { hostName = "blade-torres"; hostId = "86db1d9c"; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.185"; - prefixLength = 24; - }]; }; my.ip.tailscale = "100.92.118.36"; + my.blade.bay = 8; + my.blade.macAddress = { + internal = "e4:11:5b:ac:e3:cc"; + storage = "e4:11:5b:ac:e3:d0"; + }; services.ceph = { osd = { diff --git a/ops/nixos/blade-tuvok/default.nix b/ops/nixos/blade-tuvok/default.nix index a59f66d0db..cfcd11da5c 100644 --- a/ops/nixos/blade-tuvok/default.nix +++ b/ops/nixos/blade-tuvok/default.nix @@ -16,12 +16,24 @@ in { networking = { hostName = "blade-tuvok"; hostId = "525229f7"; - interfaces.br-ext.ipv4.addresses = [{ - address = "192.168.1.181"; - prefixLength = 24; + interfaces.en-internet.ipv4.addresses = [{ + address = "195.74.55.21"; + prefixLength = 31; }]; + interfaces.en-internet.ipv6.addresses = [{ + address = "2a03:ee40:8080:9:1::2"; + prefixLength = 126; + }]; + defaultGateway = "195.74.55.20"; + defaultGateway6 = "2a03:ee40:8080:9:1::1"; }; my.ip.tailscale = "100.119.123.33"; + my.blade.bay = 6; + my.blade.macAddress = { + internal = "e4:11:5b:ac:e3:fe"; + storage = "e4:11:5b:ac:e4:02"; + internet = "e4:11:5b:ac:e4:00"; + }; services.ceph = { mon.enable = true; diff --git a/ops/nixos/lib/blade.nix b/ops/nixos/lib/blade.nix index b3c3d17346..b3d3e6a19c 100644 --- a/ops/nixos/lib/blade.nix +++ b/ops/nixos/lib/blade.nix @@ -10,93 +10,129 @@ in { ../lib/zfs.nix ]; - boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; - boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ]; - boot.kernelParams = [ "mitigations=off" ]; - - fileSystems = let - zfs = device: { - device = device; - fsType = "zfs"; + options.my.blade = { + bay = lib.mkOption { + type = lib.types.int; }; - in { - "/" = zfs "tank/local/root"; - "/tmp" = zfs "tank/local/tmp"; - "/nix" = zfs "tank/local/nix"; - "/var" = zfs "tank/safe/var"; - "/home" = zfs "tank/safe/home"; - "/boot" = { - device = "/dev/disk/by-label/boot"; - fsType = "ext4"; + macAddress.internal = lib.mkOption { + type = lib.types.str; + }; + macAddress.storage = lib.mkOption { + type = lib.types.str; + }; + macAddress.internet = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; }; }; - boot.loader.grub.enable = true; - boot.loader.grub.version = 2; - - # Networking! - networking = { - domain = "house.as205479.net"; - nameservers = ["8.8.8.8" "8.8.4.4"]; - useDHCP = false; - bridges = let - br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; }; + config = { + boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; + boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ]; + boot.kernelParams = [ "mitigations=off" ]; + + fileSystems = let + zfs = device: { + device = device; + fsType = "zfs"; + }; in { - br-mgmt = br [ "enp4s0f0" ]; - br-storage = br [ "enp4s0f1" ]; - br-ext = br [ "enp4s0f2" ]; + "/" = zfs "tank/local/root"; + "/tmp" = zfs "tank/local/tmp"; + "/nix" = zfs "tank/local/nix"; + "/var" = zfs "tank/safe/var"; + "/home" = zfs "tank/safe/home"; + "/boot" = { + device = "/dev/disk/by-label/boot"; + fsType = "ext4"; + }; }; + + boot.loader.grub.enable = true; + boot.loader.grub.version = 2; + + # Networking! + networking = { + domain = "blade.as205479.net"; + nameservers = ["8.8.8.8" "8.8.4.4"]; + useDHCP = false; + bridges = let + br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; }; + in { + br-mgmt = br [ "en-int" ]; + br-public = br [ "vl-int-public" ]; + }; + vlans.vl-int-public = { + id = 100; + interface = "en-int"; + }; - defaultGateway = "192.168.1.5"; - - firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ]; - firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }]; - }; - - virtualisation.podman.enable = true; - - environment.systemPackages = with pkgs; [ - ceph - xfsprogs - ]; - - services.ceph = { - enable = true; - global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; - global.monHost = "192.168.1.180, 192.168.1.181, 192.168.1.184"; - global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris"; - mon.daemons = [ config.networking.hostName ]; - mds.daemons = [ config.networking.hostName ]; - rgw.daemons = [ config.networking.hostName ]; - mgr.daemons = [ config.networking.hostName ]; - mgr.enable = config.services.ceph.mon.enable; - rgw.enable = true; - }; - systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable { - enable = true; - description = "Ceph OSD pre-start"; - before = [ "network-online.target" "ceph-osd.target" ]; - wantedBy = [ "ceph-osd.target" ]; - - path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; - - serviceConfig = { - Type = "oneshot"; - ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; + interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{ + address = "10.100.0.${toString (100 + config.my.blade.bay)}"; + prefixLength = 23; + }]; + interfaces.en-storage.ipv4.addresses = lib.mkBefore [{ + address = "10.100.2.${toString (100 + config.my.blade.bay)}"; + prefixLength = 24; + }]; + + defaultGateway = lib.mkDefault "10.100.0.1"; + + firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ]; + firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }]; }; - }; - - virtualisation.libvirtd = { - enable = true; - qemuRunAsRoot = false; - qemuPackage = pkgs.qemu_full; - package = pkgs.libvirt.override { - enableCeph = true; - enableIscsi = true; + services.udev.extraRules = '' + ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int" + ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage" + '' + (lib.optionalString (config.my.blade.macAddress.internet != null) '' + ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet" + ''); + + virtualisation.podman.enable = true; + + environment.systemPackages = with pkgs; [ + ceph + xfsprogs + ]; + + services.ceph = { + enable = true; + global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c"; + global.monHost = "192.168.1.180, 192.168.1.181, 192.168.1.184"; + global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris"; + mon.daemons = [ config.networking.hostName ]; + mds.daemons = [ config.networking.hostName ]; + rgw.daemons = [ config.networking.hostName ]; + mgr.daemons = [ config.networking.hostName ]; + mgr.enable = config.services.ceph.mon.enable; + rgw.enable = true; }; + systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable { + enable = true; + description = "Ceph OSD pre-start"; + before = [ "network-online.target" "ceph-osd.target" ]; + wantedBy = [ "ceph-osd.target" ]; + + path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ]; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd"; + }; + }; + + virtualisation.libvirtd = { + enable = true; + qemuRunAsRoot = false; + qemuPackage = pkgs.qemu_full; + package = pkgs.libvirt.override { + enableCeph = true; + enableIscsi = true; + }; + }; + security.polkit.enable = true; + users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ]; + + system.stateVersion = "21.05"; }; - security.polkit.enable = true; - users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ]; - - system.stateVersion = "21.05"; }