ops/nixos: revamp blade network config

ops/nixos/blade-chakotay/default.nix

@@ -16,12 +16,13 @@ in {
   networking = {
     hostName = "blade-chakotay";
     hostId = "40bc5a75";
-    interfaces.br-ext.ipv4.addresses = [{
-      address = "192.168.1.182";
-      prefixLength = 24;
-    }];
   };
   my.ip.tailscale = "100.121.11.7";
+  my.blade.bay = 5;
+  my.blade.macAddress = {
+    internal = "e4:11:5b:ac:e3:b8";
+    storage = "e4:11:5b:ac:e3:bc";
+  };
 
   services.ceph = {
     #osd.enable = true;

ops/nixos/blade-janeway/default.nix

@@ -16,15 +16,13 @@ in {
   networking = {
     hostName = "blade-janeway";
     hostId = "3a62390f";
-    bridges.br-mgmt.interfaces = [ "enp4s0f0" ];
-    bridges.br-storage.interfaces = [ "enp4s0f1" ];
-    bridges.br-ext.interfaces = [ "enp5s0f0" ];
-    interfaces.br-ext.ipv4.addresses = [{
-      address = "192.168.1.180";
-      prefixLength = 24;
-    }];
   };
   my.ip.tailscale = "100.121.116.85";
+  my.blade.bay = 3;
+  my.blade.macAddress = {
+    internal = "e8:39:35:1f:7f:8a";
+    storage = "e8:39:35:1f:7f:8e";
+  };
 
   services.ceph = {
     mon.enable = true;

ops/nixos/blade-kim/default.nix

@@ -16,12 +16,13 @@ in {
   networking = {
     hostName = "blade-kim";
     hostId = "1643efb6";
-    interfaces.br-ext.ipv4.addresses = [{
-      address = "192.168.1.183";
-      prefixLength = 24;
-    }];
   };
   my.ip.tailscale = "100.84.36.62";
+  my.blade.bay = 1;
+  my.blade.macAddress = {
+    internal = "e4:11:5b:ac:d1:7a";
+    storage = "e4:11:5b:ac:d1:7e";
+  };
 
   services.ceph = {
     #osd.enable = true;

ops/nixos/blade-paris/default.nix

@@ -16,12 +16,35 @@ in {
   networking = {
     hostName = "blade-paris";
     hostId = "41b2a198";
-    interfaces.br-ext.ipv4.addresses = [{
+    interfaces.br-mgmt.ipv4.addresses = [{
-      address = "192.168.1.184";
+      address = "10.100.0.1";
+      prefixLength = 23;
+    }];
+    interfaces.br-public.ipv4.addresses = [{
+      address = "92.118.28.1";
       prefixLength = 24;
     }];
+    interfaces.en-internet.ipv4.addresses = [{
+      address = "192.168.1.184";
+      prefixLength = 24;
+    } {
+      address = "195.74.55.23";
+      prefixLength = 31;
+    }];
+    interfaces.en-internet.ipv6.addresses = [{
+      address = "2a03:ee40:8080:9:2::2";
+      prefixLength = 126;
+    }];
+    defaultGateway = "192.168.1.5";
+    defaultGateway6 = "2a03:ee40:8080:9:2::1";
   };
   my.ip.tailscale = "100.117.185.118";
+  my.blade.bay = 2;
+  my.blade.macAddress = {
+    internal = "e4:11:5b:ac:e4:8a";
+    storage = "e4:11:5b:ac:e4:8e";
+    internet = "e4:11:5b:ac:e4:8c";
+  };
 
   services.ceph = {
     mon.enable = true;

ops/nixos/blade-torres/default.nix

@@ -16,12 +16,13 @@ in {
   networking = {
     hostName = "blade-torres";
     hostId = "86db1d9c";
-    interfaces.br-ext.ipv4.addresses = [{
-      address = "192.168.1.185";
-      prefixLength = 24;
-    }];
   };
   my.ip.tailscale = "100.92.118.36";
+  my.blade.bay = 8;
+  my.blade.macAddress = {
+    internal = "e4:11:5b:ac:e3:cc";
+    storage = "e4:11:5b:ac:e3:d0";
+  };
 
   services.ceph = {
     osd = {

ops/nixos/blade-tuvok/default.nix

@@ -16,12 +16,24 @@ in {
   networking = {
     hostName = "blade-tuvok";
     hostId = "525229f7";
-    interfaces.br-ext.ipv4.addresses = [{
+    interfaces.en-internet.ipv4.addresses = [{
-      address = "192.168.1.181";
+      address = "195.74.55.21";
-      prefixLength = 24;
+      prefixLength = 31;
     }];
+    interfaces.en-internet.ipv6.addresses = [{
+      address = "2a03:ee40:8080:9:1::2";
+      prefixLength = 126;
+    }];
+    defaultGateway = "195.74.55.20";
+    defaultGateway6 = "2a03:ee40:8080:9:1::1";
   };
   my.ip.tailscale = "100.119.123.33";
+  my.blade.bay = 6;
+  my.blade.macAddress = {
+    internal = "e4:11:5b:ac:e3:fe";
+    storage = "e4:11:5b:ac:e4:02";
+    internet = "e4:11:5b:ac:e4:00";
+  };
 
   services.ceph = {
     mon.enable = true;

ops/nixos/lib/blade.nix

@@ -10,6 +10,23 @@ in {
     ../lib/zfs.nix
   ];
 
+  options.my.blade = {
+    bay = lib.mkOption {
+      type = lib.types.int;
+    };
+    macAddress.internal = lib.mkOption {
+      type = lib.types.str;
+    };
+    macAddress.storage = lib.mkOption {
+      type = lib.types.str;
+    };
+    macAddress.internet = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+    };
+  };
+
+  config = {
     boot.initrd.availableKernelModules = [ "ahci" "ohci_pci" "ehci_pci" "pata_atiixp" "uhci_hcd" "be2iscsi" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
     boot.kernelModules = [ "kvm-amd" "acpi_power_meter" "acpi_ipmi" "ipmi_si" ];
     boot.kernelParams = [ "mitigations=off" ];
@@ -36,22 +53,40 @@ in {
   
     # Networking!
     networking = {
-    domain = "house.as205479.net";
+      domain = "blade.as205479.net";
       nameservers = ["8.8.8.8" "8.8.4.4"];
       useDHCP = false;
       bridges = let
         br = interfaces: { interfaces = lib.mkDefault interfaces; rstp = false; };
       in {
-      br-mgmt = br [ "enp4s0f0" ];
+        br-mgmt = br [ "en-int" ];
-      br-storage = br [ "enp4s0f1" ];
+        br-public = br [ "vl-int-public" ];
-      br-ext = br [ "enp4s0f2" ];
+      };
+      vlans.vl-int-public = {
+        id = 100;
+        interface = "en-int";
       };
 
-    defaultGateway = "192.168.1.5";
+      interfaces.br-mgmt.ipv4.addresses = lib.mkBefore [{
+        address = "10.100.0.${toString (100 + config.my.blade.bay)}";
+        prefixLength = 23;
+      }];
+      interfaces.en-storage.ipv4.addresses = lib.mkBefore [{
+        address = "10.100.2.${toString (100 + config.my.blade.bay)}";
+        prefixLength = 24;
+      }];
+  
+      defaultGateway = lib.mkDefault "10.100.0.1";
   
       firewall.allowedTCPPorts = lib.mkIf config.services.ceph.enable [ 6789 3300 ];
       firewall.allowedTCPPortRanges = lib.mkIf config.services.ceph.enable [{ from = 6800; to = 7300; }];
     };
+    services.udev.extraRules = ''
+      ATTR{address}=="${config.my.blade.macAddress.internal}", NAME="en-int"
+      ATTR{address}=="${config.my.blade.macAddress.storage}", NAME="en-storage"
+    '' + (lib.optionalString (config.my.blade.macAddress.internet != null) ''
+      ATTR{address}=="${config.my.blade.macAddress.internet}", NAME="en-internet"
+    '');
   
     virtualisation.podman.enable = true;
   
@@ -99,4 +134,5 @@ in {
     users.users.lukegb.extraGroups = lib.mkAfter [ "libvirtd" ];
   
     system.stateVersion = "21.05";
+  };
 }