From 756c1a3dd2dd8ff707b50d7485132dfca92126b7 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Wed, 18 Jan 2023 21:43:48 +0000 Subject: [PATCH] cofractal-ams01: more turnup bits --- ops/nixos/cofractal-ams01/default.nix | 72 ++++++++++++--------- ops/nixos/lib/coredns/zones/db.as205479.net | 6 +- ops/nixos/lib/zfs.nix | 2 +- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/ops/nixos/cofractal-ams01/default.nix b/ops/nixos/cofractal-ams01/default.nix index e6a0528de8..4043d6502f 100644 --- a/ops/nixos/cofractal-ams01/default.nix +++ b/ops/nixos/cofractal-ams01/default.nix @@ -5,45 +5,38 @@ { depot, lib, pkgs, config, ... }: { imports = [ - ../lib/bgp.nix ../lib/zfs.nix ]; + # Otherwise _this_ machine won't enumerate things properly. + boot.zfs.devNodes = "/dev/disk/by-id"; + boot.initrd = { availableKernelModules = [ - "ehci_pci" + "nvme" + "xhci_pci" "ahci" - "usbhid" "usb_storage" + "usbhid" "sd_mod" "sr_mod" - "bnx2" # ethernet ]; - network = { - enable = true; - ssh = { - enable = true; - hostKeys = ["/persist/etc/ssh/ssh_host_ed25519_key"]; - authorizedKeys = map builtins.readFile config.users.users.lukegb.openssh.authorizedKeys.keyFiles; - }; - postCommands = '' - echo "zfs load-key -a; killall zfs" >> /root/.profile - ''; - }; }; - boot.kernelParams = [ - "ip=83.97.19.68::83.97.19.65:255.255.255.224:cofractal-ams01:eno1:none" - ]; boot.kernelModules = [ "kvm-amd" ]; + hardware.cpu.amd.updateMicrocode = true; # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; + # As of 2023-01-08, nixpkgs is pointed at Linux 5.15.86, which seems to hang with the E810-XXV NIC in this board. + boot.kernelPackages = if builtins.compareVersions pkgs.linuxPackages.kernel.version "6.0" != -1 then throw "cofractal-ams01: maybe try the stock kernel again? now at ${pkgs.linuxPackages.kernel.version}" else pkgs.linuxPackages_6_0; + boot.blacklistedKernelModules = [ "ib_core" "irdma" ]; + powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; services.zfs.rollbackOnBoot = { enable = true; - snapshot = "tank/local/root@blank"; + snapshot = "zfast/local/root@blank"; }; fileSystems = let @@ -52,18 +45,25 @@ fsType = "zfs"; }; in { - "/" = zfs "tank/local/root"; - "/nix" = zfs "tank/local/nix"; - "/tmp" = zfs "tank/local/tmp"; + "/" = zfs "zfast/local/root"; + "/nix" = zfs "zfast/local/nix"; + "/tmp" = zfs "zfast/local/tmp"; - "/persist" = zfs "tank/safe/persist"; - "/home" = zfs "tank/safe/home"; + "/persist" = zfs "zfast/safe/persist"; + "/home" = (zfs "zslow/safe/home") // { neededForBoot = true; }; "/boot" = { - device = "/dev/disk/by-partlabel/ESP"; + device = "/dev/disk/by-label/ESP"; + fsType = "vfat"; + }; + "/boot2" = { + device = "/dev/disk/by-label/ESP2"; fsType = "vfat"; }; }; + boot.loader.systemd-boot.extraInstallCommands = '' + rsync -a /boot/ /boot2/ + ''; nix.settings.max-jobs = lib.mkDefault 8; @@ -80,14 +80,26 @@ "8.8.8.8" "8.8.4.4" ]; - defaultGateway = { - address = "83.97.19.65"; + bonds.bond0 = { + interfaces = [ "enp45s0f0" "enp45s0f1" ]; + driverOptions = { + miimon = "100"; + mode = "802.3ad"; + }; }; - defaultGateway6 = { - address = "2a07:242:800:64::1"; + defaultGateway6.address = "2a09:a446:1337:ffff::1"; + interfaces.bond0 = { + ipv6.addresses = [ + { address = "2a09:a446:1337::10"; prefixLength = 64; } + { address = "2a09:a446:1337:ffff::10"; prefixLength = 120; } + ]; + ipv4.addresses = [ + { address = "199.19.152.160"; prefixLength = 30; } + ]; }; }; - # my.ip.tailscale = "100.111.191.21"; + my.ip.tailscale = "100.94.187.27"; + my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:625e:bb1b"; services.openssh.hostKeys = [ { diff --git a/ops/nixos/lib/coredns/zones/db.as205479.net b/ops/nixos/lib/coredns/zones/db.as205479.net index 5153ffa8b8..300a86d788 100644 --- a/ops/nixos/lib/coredns/zones/db.as205479.net +++ b/ops/nixos/lib/coredns/zones/db.as205479.net @@ -3,7 +3,7 @@ ; SPDX-License-Identifier: Apache-2.0 ; MNAME RNAME SERIAL REFRESH RETRY EXPIRE TTL -@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 51 600 450 3600 300 +@ 600 IN SOA frantech-lux01.as205479.net. hostmaster.lukegb.com. 52 600 450 3600 300 ; NB: this are also glue records in Google Domains. $INCLUDE tmpl.ns @@ -66,7 +66,9 @@ oracle-lon01.int 3600 IN A 100.93.85.40 oracle-lon01.int 3600 IN AAAA fd7a:115c:a1e0:ab12:4843:cd96:625d:5528 cofractal-ams01 3600 IN A 199.19.152.160 -cofractal-ams01 3600 IN AAAA 2a09:a446:1337::10 +cofractal-ams01 3600 IN AAAA 2a09:a446:1337:ffff::10 +cofractal-ams01.int 3600 IN A 100.94.187.27 +cofractal-ams01.int 3600 IN AAAA fd7a:115c:a1e0:ab12:4843:cd96:625e:bb1b blade-tuvok 3600 IN A 195.74.55.21 blade-tuvok 3600 IN AAAA 2a03:ee40:8080:9:1::2 diff --git a/ops/nixos/lib/zfs.nix b/ops/nixos/lib/zfs.nix index d8550aa920..64e2e05c85 100644 --- a/ops/nixos/lib/zfs.nix +++ b/ops/nixos/lib/zfs.nix @@ -25,7 +25,7 @@ in config = { boot.supportedFilesystems = [ "zfs" ]; - boot.zfs.devNodes = "/dev/disk/by-partuuid"; + boot.zfs.devNodes = lib.mkDefault "/dev/disk/by-partuuid"; services.zfs.autoScrub.enable = true; services.zfs.autoSnapshot = { enable = true;