cofractal-ams01: more turnup bits

ops/nixos/cofractal-ams01/default.nix

@@ -5,45 +5,38 @@
 { depot, lib, pkgs, config, ... }:
 {
   imports = [
-    ../lib/bgp.nix
     ../lib/zfs.nix
   ];
 
+  # Otherwise _this_ machine won't enumerate things properly.
+  boot.zfs.devNodes = "/dev/disk/by-id";
+
   boot.initrd = {
     availableKernelModules = [
-      "ehci_pci"
+      "nvme"
+      "xhci_pci"
       "ahci"
-      "usbhid"
       "usb_storage"
+      "usbhid"
       "sd_mod"
       "sr_mod"
-      "bnx2"  # ethernet
     ];
-    network = {
-      enable = true;
-      ssh = {
-        enable = true;
-        hostKeys = ["/persist/etc/ssh/ssh_host_ed25519_key"];
-        authorizedKeys = map builtins.readFile config.users.users.lukegb.openssh.authorizedKeys.keyFiles;
   };
-      postCommands = ''
-        echo "zfs load-key -a; killall zfs" >> /root/.profile
-      '';
-    };
-  };
-  boot.kernelParams = [
-    "ip=83.97.19.68::83.97.19.65:255.255.255.224:cofractal-ams01:eno1:none"
-  ];
   boot.kernelModules = [ "kvm-amd" ];
+  hardware.cpu.amd.updateMicrocode = true;
 
   # Use the systemd-boot EFI boot loader.
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
 
+  # As of 2023-01-08, nixpkgs is pointed at Linux 5.15.86, which seems to hang with the E810-XXV NIC in this board.
+  boot.kernelPackages = if builtins.compareVersions pkgs.linuxPackages.kernel.version "6.0" != -1 then throw "cofractal-ams01: maybe try the stock kernel again? now at ${pkgs.linuxPackages.kernel.version}" else pkgs.linuxPackages_6_0;
+  boot.blacklistedKernelModules = [ "ib_core" "irdma" ];
+
   powerManagement.cpuFreqGovernor = lib.mkDefault "performance";
   services.zfs.rollbackOnBoot = {
     enable = true;
-    snapshot = "tank/local/root@blank";
+    snapshot = "zfast/local/root@blank";
   };
 
   fileSystems = let
@@ -52,18 +45,25 @@
       fsType = "zfs";
     };
   in {
-    "/" = zfs "tank/local/root";
+    "/" = zfs "zfast/local/root";
-    "/nix" = zfs "tank/local/nix";
+    "/nix" = zfs "zfast/local/nix";
-    "/tmp" = zfs "tank/local/tmp";
+    "/tmp" = zfs "zfast/local/tmp";
 
-    "/persist" = zfs "tank/safe/persist";
+    "/persist" = zfs "zfast/safe/persist";
-    "/home" = zfs "tank/safe/home";
+    "/home" = (zfs "zslow/safe/home") // { neededForBoot = true; };
 
     "/boot" = {
-      device = "/dev/disk/by-partlabel/ESP";
+      device = "/dev/disk/by-label/ESP";
+      fsType = "vfat";
+    };
+    "/boot2" = {
+      device = "/dev/disk/by-label/ESP2";
       fsType = "vfat";
     };
   };
+  boot.loader.systemd-boot.extraInstallCommands = ''
+    rsync -a /boot/ /boot2/
+  '';
 
   nix.settings.max-jobs = lib.mkDefault 8;
 
@@ -80,14 +80,26 @@
       "8.8.8.8"
       "8.8.4.4"
     ];
-    defaultGateway = {
+    bonds.bond0 = {
-      address = "83.97.19.65";
+      interfaces = [ "enp45s0f0" "enp45s0f1" ];
-    };
+      driverOptions = {
-    defaultGateway6 = {
+        miimon = "100";
-      address = "2a07:242:800:64::1";
+        mode = "802.3ad";
       };
     };
-  # my.ip.tailscale = "100.111.191.21";
+    defaultGateway6.address = "2a09:a446:1337:ffff::1";
+    interfaces.bond0 = {
+      ipv6.addresses = [
+        { address = "2a09:a446:1337::10"; prefixLength = 64; }
+        { address = "2a09:a446:1337:ffff::10"; prefixLength = 120; }
+      ];
+      ipv4.addresses = [
+        { address = "199.19.152.160"; prefixLength = 30; }
+      ];
+    };
+  };
+  my.ip.tailscale = "100.94.187.27";
+  my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:625e:bb1b";
 
   services.openssh.hostKeys = [
     {

ops/nixos/lib/coredns/zones/db.as205479.net

@@ -3,7 +3,7 @@
 ; SPDX-License-Identifier: Apache-2.0
 
 ;              MNAME                        RNAME                  SERIAL REFRESH RETRY EXPIRE TTL
-@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 51     600     450   3600   300
+@ 600 IN SOA   frantech-lux01.as205479.net. hostmaster.lukegb.com. 52     600     450   3600   300
 
 ; NB: this are also glue records in Google Domains.
 $INCLUDE tmpl.ns
@@ -66,7 +66,9 @@ oracle-lon01.int    3600 IN A    100.93.85.40
 oracle-lon01.int    3600 IN AAAA fd7a:115c:a1e0:ab12:4843:cd96:625d:5528
 
 cofractal-ams01     3600 IN A    199.19.152.160
-cofractal-ams01     3600 IN AAAA 2a09:a446:1337::10
+cofractal-ams01     3600 IN AAAA 2a09:a446:1337:ffff::10
+cofractal-ams01.int 3600 IN A    100.94.187.27
+cofractal-ams01.int 3600 IN AAAA fd7a:115c:a1e0:ab12:4843:cd96:625e:bb1b
 
 blade-tuvok         3600 IN A    195.74.55.21
 blade-tuvok         3600 IN AAAA 2a03:ee40:8080:9:1::2

ops/nixos/lib/zfs.nix

@@ -25,7 +25,7 @@ in
 
   config = {
     boot.supportedFilesystems = [ "zfs" ];
-    boot.zfs.devNodes = "/dev/disk/by-partuuid";
+    boot.zfs.devNodes = lib.mkDefault "/dev/disk/by-partuuid";
     services.zfs.autoScrub.enable = true;
     services.zfs.autoSnapshot = {
       enable = true;