# SPDX-FileCopyrightText: 2024 Luke Granger-Brown # # SPDX-License-Identifier: Apache-2.0 { depot, lib, pkgs, config, ... }: { imports = [ ../lib/zfs.nix ./bgp.nix ../lib/bgp.nix ../lib/gitlab-runner-cacher.nix #../lib/nixbuild-distributed.nix # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid ../lib/hackyplayer.nix ../lib/emfminiserv.nix ../lib/seaweedfs.nix ../lib/fup.nix ./bsky-pds.nix ]; # Otherwise _this_ machine won't enumerate things properly. boot.zfs.devNodes = "/dev/disk/by-id"; boot.initrd = { availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ]; systemd.enable = true; }; security.tpm2.enable = true; boot.kernelModules = [ "kvm-amd" ]; hardware.cpu.amd.updateMicrocode = true; boot.kernelParams = [ "nomodeset" ]; environment.systemPackages = with pkgs; [ clevis ]; # Use the systemd-boot EFI boot loader. boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; powerManagement.cpuFreqGovernor = lib.mkDefault "performance"; fileSystems = let zfs = device: { device = device; fsType = "zfs"; }; in { "/" = zfs "zboot/local/root"; "/nix" = zfs "zboot/local/nix"; "/persist" = zfs "zboot/safe/persist"; "/store" = zfs "zu2/safe/store"; "/home" = zfs "zu2/safe/home"; "/store/emf" = zfs "zu2/safe/store/emf"; "/store/emf/2024" = zfs "zu2/safe/store/emf/2024"; "/store/emf/2024/video" = zfs "zu2/safe/store/emf/2024/video"; "/store/emf/2024/video/source" = zfs "zu2/safe/store/emf/2024/video/source"; "/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input"; "/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output"; "/var/lib/private/seaweedfs-master" = zfs "zu2/safe/var/lib/private/seaweedfs-master"; "/var/lib/private/seaweedfs-filer" = zfs "zu2/safe/var/lib/private/seaweedfs-filer"; "/var/lib/private/seaweedfs-volume" = zfs "zu2/safe/var/lib/private/seaweedfs-volume"; "/var/lib/private/seaweedfs-volume/data" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/data"; "/var/lib/private/seaweedfs-volume/idx" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/idx"; "/boot" = { device = "/dev/disk/by-label/ESP"; fsType = "vfat"; }; "/boot2" = { device = "/dev/disk/by-label/ESP2"; fsType = "vfat"; }; }; boot.loader.systemd-boot.extraInstallCommands = '' rsync -a /boot/ /boot2/ ''; nix.settings.max-jobs = lib.mkDefault 64; # Networking! networking = { hostName = "rexxar"; domain = "as205479.net"; hostId = "b46c2ae9"; useNetworkd = true; firewall = { allowedUDPPorts = [ 51821 51822 51823 34197 # factorio 443 3784 # BFD ]; allowedTCPPorts = [ 80 443 ]; }; }; systemd.network = let wireguard = { name, listenPort, privateKey, publicKey, endpoint ? null }: { netdevConfig = { Name = name; Kind = "wireguard"; Description = "WireGuard tunnel ${name}"; }; wireguardConfig = { ListenPort = listenPort; PrivateKeyFile = privateKey; }; wireguardPeers = [(lib.mkMerge [{ PublicKey = publicKey; AllowedIPs = [ "0.0.0.0/0" "::/0" ]; } (lib.mkIf (endpoint != null) { Endpoint = endpoint; })])]; }; swannWireguard = args: wireguard (args // { privateKey = config.my.vault.secrets.wg-swann-private.path; publicKey = "XyfovUP6GUwIg15t5UWxicfxooeto/U/7nLs7Zu8HH4="; }); cofractalWireguard = args: wireguard (args // { privateKey = config.my.vault.secrets.wg-cofractal-ams01-private.path; publicKey = "qKi6mWIhV2n16LGH16Iug5W+Bx4Fx7eprxCgA/1Ra1g="; }); in { netdevs."40-wg-swann-ee" = swannWireguard { name = "wg-swann-ee"; listenPort = 51821; }; netdevs."40-wg-swann-gnet" = swannWireguard { name = "wg-swann-gnet"; listenPort = 51822; endpoint = "185.250.189.20:51822"; }; netdevs."40-wg-cofractal" = cofractalWireguard { name = "wg-cofractal"; listenPort = 51823; endpoint = "[2a09:a446:1337:ffff::10]:51823"; }; networks."40-wg-swann-ee" = { matchConfig.Name = "wg-swann-ee"; address = [ "92.118.30.3/31" "2a09:a442::2:2/112" ]; }; networks."40-wg-swann-gnet" = { matchConfig.Name = "wg-swann-gnet"; address = [ "92.118.30.5/31" "2a09:a442::3:2/112" ]; }; networks."40-wg-cofractal" = { matchConfig.Name = "wg-cofractal"; address = [ "169.254.200.0/31" ]; }; networks."10-usb0" = { matchConfig.Name = "usb0"; address = [ "169.254.0.1/24" # IPMI 169.254.0.17 ]; }; networks."10-enp193s0f0np0" = { matchConfig.Name = "enp193s0f0np0"; # Telia address = [ "62.115.150.105/31" "2001:2035:0:1f7e::2/126" ]; }; networks."10-enp193s0f1np1" = { matchConfig.Name = "enp193s0f1np1"; networkConfig.VLAN = [ "vl-velox1" ]; }; # enp9s0f0 - previously velox copper cable 1, now enp193s0f1np1 via 10G networks."10-enp9s0f1" = { matchConfig.Name = "enp9s0f1"; networkConfig.VLAN = [ "vl-velox2" "vl-linx" ]; # also IPMI, vlan 300 }; netdevs."20-vl-velox1" = { netdevConfig = { Name = "vl-velox1"; Kind = "vlan"; MACAddress = "8C:1F:64:0B:6F:00"; }; vlanConfig = { Id = 100; }; }; networks."20-vl-velox1" = { matchConfig.Name = "vl-velox1"; address = [ "195.74.55.21/31" "2a03:ee40:8080:9:1::2/126" ]; networkConfig.DNS = [ "2001:4860:4860::8888" "2001:4860:4860::8844" "8.8.8.8" "8.8.4.4" "1.1.1.1" ]; networkConfig.DNSDefaultRoute = true; routes = [{ Gateway = "195.74.55.20"; } { Gateway = "2a03:ee40:8080:9:1::1"; }]; }; netdevs."20-vl-velox2" = { netdevConfig = { Name = "vl-velox2"; Kind = "vlan"; MACAddress = "8C:1F:64:0B:6F:01"; }; vlanConfig = { Id = 100; }; }; networks."20-vl-velox2" = { matchConfig.Name = "vl-velox2"; address = [ "195.74.55.23/31" "2a03:ee40:8080:9:2::2/126" ]; networkConfig.DNS = [ "2001:4860:4860::8888" "2001:4860:4860::8844" "8.8.8.8" "8.8.4.4" "1.1.1.1" ]; networkConfig.DNSDefaultRoute = true; routes = [{ Gateway = "195.74.55.22"; } { Gateway = "2a03:ee40:8080:9:2::1"; }]; }; netdevs."20-vl-linx" = { netdevConfig = { Name = "vl-linx"; Kind = "vlan"; MACAddress = "8C:1F:64:0B:6F:02"; }; vlanConfig = { Id = 200; }; }; networks."20-vl-linx" = { matchConfig.Name = "vl-linx"; address = [ "195.66.224.58/21" "2001:7f8:4::3:22a7:1/48" ]; networkConfig = { IPv6LinkLocalAddressGenerationMode = "eui64"; LLMNR = false; MulticastDNS = false; IPv6AcceptRA = false; IPv4ProxyARP = false; IPv6ProxyNDP = false; IPv6SendRA = false; }; }; networks."60-lo" = { matchConfig.Name = "lo"; addresses = [{ Address = "127.0.0.1/8"; Scope = "host"; } { Address = "::1/128"; } { Address = "92.118.30.251/32"; } { Address = "2a09:a442:1000::/128"; }]; }; }; my.ip.tailscale = "100.97.110.48"; my.ip.tailscale6 = "fd7a:115c:a1e0::3a01:6e30"; #my.coredns.bind = [ "bond0" "tailscale0" "127.0.0.1" "::1" ]; services.openssh.hostKeys = [ { path = "/persist/etc/ssh/ssh_host_ed25519_key"; type = "ed25519"; } { path = "/persist/etc/ssh/ssh_host_rsa_key"; type = "rsa"; bits = 4096; } ]; systemd.mounts = let bindMount' = dir: { unitConfig.RequiresMountsFor = dir; options = "bind"; what = "/persist${dir}"; where = dir; }; bindMountSvc = dir: svc: (bindMount' dir) // { requiredBy = [svc]; before = [svc]; wantedBy = ["multi-user.target"]; }; bindMountSvcDynamic = dir: svc: (bindMount' "/var/lib/private/${dir}") // { requiredBy = [svc]; before = [svc]; wantedBy = ["multi-user.target"]; }; bindMount = dir: (bindMount' dir) // { wantedBy = ["multi-user.target"]; }; in [ (bindMountSvc "/var/lib/tailscale" "tailscaled.service") (bindMountSvc "/var/lib/libvirt" "libvirt.service") (bindMountSvc "/var/lib/private/factorio" "factorio.service") ]; boot.kernel.sysctl = { "net.ipv4.ip_forward" = 1; "net.ipv6.conf.all.forwarding" = 1; "net.ipv4.conf.vl-linx.arp_announce" = 1; "net.ipv4.conf.vl-linx.arp_ignore" = 1; "net.ipv4.neigh.vl-linx.base_reachable_time_ms" = 14400000; "net.ipv6.neigh.vl-linx.base_reachable_time_ms" = 14400000; "net.ipv4.neigh.default.gc_thresh1" = 2048; "net.ipv6.neigh.default.gc_thresh1" = 2048; "net.ipv4.neigh.default.gc_thresh2" = 4096; "net.ipv6.neigh.default.gc_thresh2" = 4096; "net.ipv4.neigh.default.gc_thresh3" = 8192; "net.ipv6.neigh.default.gc_thresh3" = 8192; }; boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; my.vault.secrets = let wireguardSecret = key: { group = "systemd-network"; template = '' {{- with secret "kv/apps/wireguard/rexxar" -}} {{- .Data.data.${key} -}} {{- end -}} ''; }; in { wg-swann-private = wireguardSecret "privateKeyToSwann"; wg-cofractal-ams01-private = wireguardSecret "privateKeyToCofractalAms01"; }; users.users.lukegb.extraGroups = lib.mkAfter [ "hackyplayer" ]; users.users.samw = { isNormalUser = true; extraGroups = lib.mkAfter [ "hackyplayer" ]; openssh.authorizedKeys.keys = [ "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDQdGzwYiallvWXIHgSAf2GOwMJKA8bxPmwyuO+vsd1HwB65hMRPCpKS+FNLIpkrADNnuhGS3xGCGSSuQ+zAu/g=" "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNLrJyUXmiFWb9vhlTWZLYr64IsKut+c9TGqq3/uwPDeF4X0Qb2jzxqXfQcDSztjR09JHbC8BOqfpYYT9LHahIo=" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOOicoYtzJ3vXSkY7MkriQrfyZ/gP8ONM2OHhO0zi2cl" "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGUoi76VYnqsPZ4IjyfNb8NfaaAfcU9iAbARFEfYTuNwxmYUjv+RpCBC1Gqi0YpdONyrWi9XljjAtY92P4jZffQ=" ]; }; services.factorio = { inherit (depot.ops.secrets.factorio) username token; enable = true; saveName = "lukegb20241107-spaceage"; game-name = "Briefcase Full of Bees"; #mods = depot.nix.pkgs.factorio-mods._all; #mods-dat = ./mod-settings.dat; admins = ["lukegb"]; package = pkgs.factorio-headless-experimental; extraSettings = { auto_pause = true; only_admins_can_pause_the_game = false; game_password = depot.ops.secrets.factorioServerPassword; non_blocking_saving = true; autosave_only_on_server = true; autosave_interval = 5; autosave_slots = 60; }; }; my.hackyplayer = { enable = true; hostname = "hackyplayer.voc.emf.camp"; workerConcurrency = 13; }; my.emfminiserv = { enable = true; hostname = "prerelease.voc.emf.camp"; }; hardware.rasdaemon.enable = true; services.prometheus.exporters.ipmi = { enable = true; }; my.services.seaweedfs = { securitySettings = { cors.allowed_origins.values = "*"; }; master = { enable = true; options = { port = 21000; ip = "rexxar.int.as205479.net"; "ip.bind" = config.my.ip.tailscale6; mdir = "/var/lib/seaweedfs-master/metadata"; defaultReplication = "000"; }; }; filer = { enable = true; earlyOptions = { v = 4; }; options = { port = 21010; ip = "rexxar.int.as205479.net"; "ip.bind" = config.my.ip.tailscale6; master = "[${config.my.ip.tailscale6}]:21000"; encryptVolumeData = true; s3 = true; "s3.domainName" = "objdump.zxcvbnm.ninja"; "s3.localSocket" = "/run/seaweedfs-filer/s3.sock"; "s3.port" = 21012; webdav = true; "webdav.port" = 21014; iam = true; "iam.ip" = "127.0.0.1"; "iam.port" = 21015; defaultReplicaPlacement = "000"; }; settings = { leveldb2 = { enabled = true; dir = "/var/lib/seaweedfs-filer/leveldb2"; }; }; }; volume = { enable = true; earlyOptions = { v = 4; }; options = { port = 21100; ip = "rexxar.int.as205479.net"; "ip.bind" = config.my.ip.tailscale6; dataCenter = "ixn-lon1"; rack = "vlx-lukegb1"; max = 0; mserver = "[${config.my.ip.tailscale6}]:21000"; }; }; cli.settings = { cluster.default = "rexxar"; cluster.rexxar = { master = "[${config.my.ip.tailscale6}]:21000"; }; }; }; my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = { hostnames = [ "objdump.zxcvbnm.ninja" "*.objdump.zxcvbnm.ninja" ]; reloadOrRestartUnits = [ "caddy.service" ]; }; services.caddy = { enable = true; virtualHosts."objdump.zxcvbnm.ninja" = { serverAliases = [ "*.objdump.zxcvbnm.ninja" ]; extraConfig = '' tls /var/lib/acme/objdump.zxcvbnm.ninja/fullchain.pem /var/lib/acme/objdump.zxcvbnm.ninja/privkey.pem reverse_proxy unix//run/seaweedfs-filer/s3.sock ''; }; virtualHosts."http://objdump.zxcvbnm.ninja" = { serverAliases = [ "http://*.objdump.zxcvbnm.ninja" ]; extraConfig = '' reverse_proxy unix//run/seaweedfs-filer/s3.sock ''; }; }; systemd.services.caddy.serviceConfig = { SupplementaryGroups = lib.mkAfter [ "acme" ]; ReadOnlyPaths = lib.mkAfter [ "/var/lib/acme" ]; }; my.fup.listen = []; system.stateVersion = "24.05"; }