lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

ops/nixos: revamp blade network config

Browse source
This commit is contained in:
Luke Granger-Brown 2021-03-12 14:47:08 +00:00
parent 75ca762b89
commit 53b7ca1c8a
7 changed files with 175 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
ops/nixos/blade-chakotay/default.nix
View file

@ -16,12 +16,13 @@ in {
networking = { networking = {
hostName = "blade-chakotay"; hostName = "blade-chakotay";
hostId = "40bc5a75"; hostId = "40bc5a75";
interfaces.br-ext.ipv4.addresses = [{
address = "192.168.1.182";
prefixLength = 24;
}];
}; };
my.ip.tailscale = "100.121.11.7"; my.ip.tailscale = "100.121.11.7";
my.blade.bay = 5;
my.blade.macAddress = {
internal = "e4:11:5b:ac:e3:b8";
storage = "e4:11:5b:ac:e3:bc";
};
services.ceph = { services.ceph = {
#osd.enable = true; #osd.enable = true;

12
ops/nixos/blade-janeway/default.nix
View file

@ -16,15 +16,13 @@ in {
networking = { networking = {
hostName = "blade-janeway"; hostName = "blade-janeway";
hostId = "3a62390f"; hostId = "3a62390f";
bridges.br-mgmt.interfaces = [ "enp4s0f0" ];
bridges.br-storage.interfaces = [ "enp4s0f1" ];
bridges.br-ext.interfaces = [ "enp5s0f0" ];
interfaces.br-ext.ipv4.addresses = [{
address = "192.168.1.180";
prefixLength = 24;
}];
}; };
my.ip.tailscale = "100.121.116.85"; my.ip.tailscale = "100.121.116.85";
my.blade.bay = 3;
my.blade.macAddress = {
internal = "e8:39:35:1f:7f:8a";
storage = "e8:39:35:1f:7f:8e";
};
services.ceph = { services.ceph = {
mon.enable = true; mon.enable = true;

9
ops/nixos/blade-kim/default.nix
View file

@ -16,12 +16,13 @@ in {
networking = { networking = {
hostName = "blade-kim"; hostName = "blade-kim";
hostId = "1643efb6"; hostId = "1643efb6";
interfaces.br-ext.ipv4.addresses = [{
address = "192.168.1.183";
prefixLength = 24;
}];
}; };
my.ip.tailscale = "100.84.36.62"; my.ip.tailscale = "100.84.36.62";
my.blade.bay = 1;
my.blade.macAddress = {
internal = "e4:11:5b:ac:d1:7a";
storage = "e4:11:5b:ac:d1:7e";
};
services.ceph = { services.ceph = {
#osd.enable = true; #osd.enable = true;

27
ops/nixos/blade-paris/default.nix
View file

@ -16,12 +16,35 @@ in {
networking = { networking = {
hostName = "blade-paris"; hostName = "blade-paris";
hostId = "41b2a198"; hostId = "41b2a198";
interfaces.br-ext.ipv4.addresses = [{ interfaces.br-mgmt.ipv4.addresses = [{
address = "192.168.1.184"; address = "10.100.0.1";
prefixLength = 23;
}];
interfaces.br-public.ipv4.addresses = [{
address = "92.118.28.1";
prefixLength = 24; prefixLength = 24;
}]; }];
interfaces.en-internet.ipv4.addresses = [{
address = "192.168.1.184";
prefixLength = 24;
} {
address = "195.74.55.23";
prefixLength = 31;
}];
interfaces.en-internet.ipv6.addresses = [{
address = "2a03:ee40:8080:9:2::2";
prefixLength = 126;
}];
defaultGateway = "192.168.1.5";
defaultGateway6 = "2a03:ee40:8080:9:2::1";
}; };
my.ip.tailscale = "100.117.185.118"; my.ip.tailscale = "100.117.185.118";
my.blade.bay = 2;
my.blade.macAddress = {
internal = "e4:11:5b:ac:e4:8a";
storage = "e4:11:5b:ac:e4:8e";
internet = "e4:11:5b:ac:e4:8c";
};
services.ceph = { services.ceph = {
mon.enable = true; mon.enable = true;

9
ops/nixos/blade-torres/default.nix
View file

@ -16,12 +16,13 @@ in {
networking = { networking = {
hostName = "blade-torres"; hostName = "blade-torres";
hostId = "86db1d9c"; hostId = "86db1d9c";
interfaces.br-ext.ipv4.addresses = [{
address = "192.168.1.185";
prefixLength = 24;
}];
}; };
my.ip.tailscale = "100.92.118.36"; my.ip.tailscale = "100.92.118.36";
my.blade.bay = 8;
my.blade.macAddress = {
internal = "e4:11:5b:ac:e3:cc";
storage = "e4:11:5b:ac:e3:d0";
};
services.ceph = { services.ceph = {
osd = { osd = {

18
ops/nixos/blade-tuvok/default.nix
View file

@ -16,12 +16,24 @@ in {
networking = { networking = {
hostName = "blade-tuvok"; hostName = "blade-tuvok";
hostId = "525229f7"; hostId = "525229f7";
interfaces.br-ext.ipv4.addresses = [{ interfaces.en-internet.ipv4.addresses = [{
address = "192.168.1.181"; address = "195.74.55.21";
prefixLength = 24; prefixLength = 31;
}]; }];
interfaces.en-internet.ipv6.addresses = [{
address = "2a03:ee40:8080:9:1::2";
prefixLength = 126;
}];
defaultGateway = "195.74.55.20";
defaultGateway6 = "2a03:ee40:8080:9:1::1";
}; };
my.ip.tailscale = "100.119.123.33"; my.ip.tailscale = "100.119.123.33";
my.blade.bay = 6;
my.blade.macAddress = {
internal = "e4:11:5b:ac:e3:fe";
storage = "e4:11:5b:ac:e4:02";
internet = "e4:11:5b:ac:e4:00";
};
services.ceph = { services.ceph = {
mon.enable = true; mon.enable = true;

182
ops/nixos/lib/blade.nix
View file

@ -10,93 +10,129 @@ in {
../lib/zfs.nix ../lib/zfs.nix
]; ];
boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; options.my.blade = {
boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ]; bay = lib.mkOption {
boot.kernelParams = [ "mitigations=off" ]; type = lib.types.int;
fileSystems = let
zfs = device: {
device = device;
fsType = "zfs";
}; };
in { macAddress.internal = lib.mkOption {
"/" = zfs "tank/local/root"; type = lib.types.str;
"/tmp" = zfs "tank/local/tmp"; };
"/nix" = zfs "tank/local/nix"; macAddress.storage = lib.mkOption {
"/var" = zfs "tank/safe/var"; type = lib.types.str;
"/home" = zfs "tank/safe/home"; };
"/boot" = { macAddress.internet = lib.mkOption {
device = "/dev/disk/by-label/boot"; type = lib.types.nullOr lib.types.str;
fsType = "ext4"; default = null;
}; };
}; };
boot.loader.grub.enable = true; config = {
boot.loader.grub.version = 2; boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];
boot.kernelParams = [ "mitigations=off" ];
# Networking! fileSystems = let
networking = { zfs = device: {
domain = "house.as205479.net"; device = device;
nameservers = ["8.8.8.8" "8.8.4.4"]; fsType = "zfs";
useDHCP = false; };
bridges = let
br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
in { in {
br-mgmt = br [ "enp4s0f0" ]; "/" = zfs "tank/local/root";
br-storage = br [ "enp4s0f1" ]; "/tmp" = zfs "tank/local/tmp";
br-ext = br [ "enp4s0f2" ]; "/nix" = zfs "tank/local/nix";
"/var" = zfs "tank/safe/var";
"/home" = zfs "tank/safe/home";
"/boot" = {
device = "/dev/disk/by-label/boot";
fsType = "ext4";
};
}; };
defaultGateway = "192.168.1.5"; boot.loader.grub.enable = true;
boot.loader.grub.version = 2;
firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ]; # Networking!
firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }]; networking = {
}; domain = "blade.as205479.net";
nameservers = ["8.8.8.8" "8.8.4.4"];
useDHCP = false;
bridges = let
br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
in {
br-mgmt = br [ "en-int" ];
br-public = br [ "vl-int-public" ];
};
vlans.vl-int-public = {
id = 100;
interface = "en-int";
};
virtualisation.podman.enable = true; interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
address = "10.100.0.${toString (100 + config.my.blade.bay)}";
prefixLength = 23;
}];
interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
address = "10.100.2.${toString (100 + config.my.blade.bay)}";
prefixLength = 24;
}];
environment.systemPackages = with pkgs; [ defaultGateway = lib.mkDefault "10.100.0.1";
ceph
xfsprogs
];
services.ceph = { firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
enable = true; firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];
global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
global.monHost = "192.168.1.180, 192.168.1.181, 192.168.1.184";
global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
mon.daemons = [ config.networking.hostName ];
mds.daemons = [ config.networking.hostName ];
rgw.daemons = [ config.networking.hostName ];
mgr.daemons = [ config.networking.hostName ];
mgr.enable = config.services.ceph.mon.enable;
rgw.enable = true;
};
systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
enable = true;
description = "Ceph OSD pre-start";
before = [ "network-online.target" "ceph-osd.target" ];
wantedBy = [ "ceph-osd.target" ];
path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
}; };
}; services.udev.extraRules = ''
ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
'' + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
'');
virtualisation.libvirtd = { virtualisation.podman.enable = true;
enable = true;
qemuRunAsRoot = false; environment.systemPackages = with pkgs; [
qemuPackage = pkgs.qemu_full; ceph
package = pkgs.libvirt.override { xfsprogs
enableCeph = true; ];
enableIscsi = true;
services.ceph = {
enable = true;
global.fsid = "521a59a5-a597-4432-b248-1ecd3c76ca4c";
global.monHost = "192.168.1.180, 192.168.1.181, 192.168.1.184";
global.monInitialMembers = "blade-janeway, blade-tuvok, blade-paris";
mon.daemons = [ config.networking.hostName ];
mds.daemons = [ config.networking.hostName ];
rgw.daemons = [ config.networking.hostName ];
mgr.daemons = [ config.networking.hostName ];
mgr.enable = config.services.ceph.mon.enable;
rgw.enable = true;
}; };
}; systemd.services.ceph-osd-lvm-activate = lib.mkIf config.services.ceph.osd.enable {
security.polkit.enable = true; enable = true;
users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ]; description = "Ceph OSD pre-start";
before = [ "network-online.target" "ceph-osd.target" ];
wantedBy = [ "ceph-osd.target" ];
system.stateVersion = "21.05"; path = [ pkgs.lvm2.bin pkgs.util-linux pkgs.coreutils ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.ceph.out}/bin/ceph-volume lvm activate --all --no-systemd";
};
};
virtualisation.libvirtd = {
enable = true;
qemuRunAsRoot = false;
qemuPackage = pkgs.qemu_full;
package = pkgs.libvirt.override {
enableCeph = true;
enableIscsi = true;
};
};
security.polkit.enable = true;
users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];
system.stateVersion = "21.05";
};
} }