diff --git a/ops/nixos/etheroute-lon01/default.nix b/ops/nixos/etheroute-lon01/default.nix index c8ee245911..06d7a80443 100644 --- a/ops/nixos/etheroute-lon01/default.nix +++ b/ops/nixos/etheroute-lon01/default.nix @@ -340,9 +340,6 @@ allow_websockets = true; timeout = "0"; }) - (service "blade-tuvok.int.as205479.net:7480" "objdump.zxcvbnm.ninja" (public { - timeout = "30m"; # Uploads can take a while; bump the timeout. - })) (secureService "totoro.int.as205479.net" "invoices.lukegb.com" (public { regex = "^/((third_party|ajax|client_area|pdf)/.*|[a-zA-Z0-9]{8})$"; tls_skip_verify = true; @@ -403,7 +400,6 @@ "lukegb.com" "*.lukegb.com" "*.int.lukegb.com" - "objdump.zxcvbnm.ninja" ]; reloadOrRestartUnits = [ "pomerium.service" ]; }; diff --git a/ops/nixos/lib/seaweedfs.nix b/ops/nixos/lib/seaweedfs.nix new file mode 100644 index 0000000000..aec63a53c3 --- /dev/null +++ b/ops/nixos/lib/seaweedfs.nix @@ -0,0 +1,209 @@ +{ depot, config, pkgs, lib, ... }: + +let + cfg = config.my.services.seaweedfs; + + tomlFormat = pkgs.formats.toml {}; + + makeCommandLine' = prefix: options: let + makeFlag = name: value: + if builtins.isAttrs value then makeCommandLine' "${prefix}${name}." value + else if builtins.isString value then "-${prefix}${name}=${lib.strings.escapeShellArg value}" + else if builtins.isInt value then "-${prefix}${name}=${toString value}" + else "-${name}"; + optionsList = lib.attrsets.mapAttrsToList makeFlag options; + in + lib.strings.concatStringsSep " " optionsList; + makeCommandLine = makeCommandLine' ""; + + commandLineType = lib.types.oneOf [ + lib.types.bool + lib.types.int + lib.types.str + (lib.types.attrsOf commandLineType) + ]; +in { + options.my.services.seaweedfs = { + package = lib.mkOption { + type = lib.types.package; + default = pkgs.seaweedfs; + }; + + securitySettings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + + filer = { + enable = lib.mkEnableOption "SeaweedFS filer"; + + package = lib.mkOption { + type = lib.types.package; + }; + + options = lib.mkOption { + type = lib.types.submodule { + freeformType = commandLineType; + }; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + notificationSettings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + replicationSettings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + }; + + master = { + enable = lib.mkEnableOption "SeaweedFS master"; + + package = lib.mkOption { + type = lib.types.package; + }; + + options = lib.mkOption { + type = lib.types.submodule { + freeformType = commandLineType; + }; + default = {}; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + }; + + volume = { + enable = lib.mkEnableOption "SeaweedFS volume"; + + package = lib.mkOption { + type = lib.types.package; + }; + + options = lib.mkOption { + type = lib.types.submodule { + freeformType = commandLineType; + }; + default = {}; + }; + }; + + cli = { + enable = lib.mkEnableOption "add SeaweedFS CLI"; + + package = lib.mkOption { + type = lib.types.package; + }; + + settings = lib.mkOption { + type = lib.types.submodule { + freeformType = tomlFormat.type; + }; + default = {}; + }; + }; + }; + + config = lib.mkMerge [{ + my.services.seaweedfs.master.package = lib.mkDefault cfg.package; + my.services.seaweedfs.filer.package = lib.mkDefault cfg.package; + my.services.seaweedfs.volume.package = lib.mkDefault cfg.package; + my.services.seaweedfs.cli.package = lib.mkDefault cfg.package; + } (lib.mkIf (cfg.filer.enable || cfg.master.enable || cfg.volume.enable) { + environment.etc."seaweedfs/security.toml".source = tomlFormat.generate "seaweedfs-security.toml" cfg.securitySettings; + + my.services.seaweedfs.cli.enable = lib.mkDefault true; + + users.groups.seaweedfs = {}; + + systemd.targets.seaweedfs = { + description = "SeaweedFS components"; + wantedBy = [ "multi-user.target" ]; + }; + }) (lib.mkIf (cfg.cli.enable) { + environment.systemPackages = [ cfg.cli.package ]; + + environment.etc."seaweedfs/shell.toml".source = tomlFormat.generate "seaweedfs-shell.toml" cfg.cli.settings; + }) (lib.mkIf (cfg.filer.enable) { + environment.etc."seaweedfs/filer.toml".source = tomlFormat.generate "seaweedfs-filer.toml" cfg.filer.settings; + environment.etc."seaweedfs/notification.toml".source = tomlFormat.generate "seaweedfs-notification.toml" cfg.filer.notificationSettings; + environment.etc."seaweedfs/replication.toml".source = tomlFormat.generate "seaweedfs-replication.toml" cfg.filer.replicationSettings; + + my.services.seaweedfs.filer.options = { + localSocket = lib.mkDefault "/run/seaweedfs-filer/filer.sock"; + }; + + systemd.services."seaweedfs-filer" = { + wantedBy = [ "seaweedfs.target" ]; + restartTriggers = [ + (config.environment.etc."seaweedfs/security.toml".source) + (config.environment.etc."seaweedfs/filer.toml".source) + (config.environment.etc."seaweedfs/notification.toml".source) + (config.environment.etc."seaweedfs/replication.toml".source) + ]; + serviceConfig = { + ExecStart = "${cfg.filer.package}/bin/weed filer ${makeCommandLine cfg.filer.options}"; + User = "seaweedfs-filer"; + Group = "seaweedfs"; + DynamicUser = true; + RuntimeDirectory = "seaweedfs-filer"; + StateDirectory = "seaweedfs-filer"; + }; + }; + }) (lib.mkIf (cfg.master.enable) { + environment.etc."seaweedfs/master.toml".source = tomlFormat.generate "seaweedfs-master.toml" cfg.master.settings; + + systemd.services."seaweedfs-master" = { + wantedBy = [ "seaweedfs.target" ]; + restartTriggers = [ + (config.environment.etc."seaweedfs/security.toml".source) + ]; + serviceConfig = { + ExecStart = "${cfg.master.package}/bin/weed master ${makeCommandLine cfg.master.options}"; + User = "seaweedfs-master"; + Group = "seaweedfs"; + DynamicUser = true; + RuntimeDirectory = "seaweedfs-master"; + StateDirectory = "seaweedfs-master"; + }; + }; + }) (lib.mkIf (cfg.volume.enable) { + my.services.seaweedfs.volume.options = { + dir = lib.mkDefault "/var/lib/seaweedfs-volume/data"; + "dir.idx" = lib.mkDefault "/var/lib/seaweedfs-volume/idx"; + }; + + systemd.services."seaweedfs-volume" = { + wantedBy = [ "seaweedfs.target" ]; + restartTriggers = [ + (config.environment.etc."seaweedfs/security.toml".source) + ]; + serviceConfig = { + ExecStart = "${cfg.volume.package}/bin/weed volume ${makeCommandLine cfg.volume.options}"; + User = "seaweedfs-volume"; + Group = "seaweedfs"; + DynamicUser = true; + RuntimeDirectory = "seaweedfs-volume"; + StateDirectory = "seaweedfs-volume"; + }; + }; + })]; +} diff --git a/ops/nixos/lib/secretsmgr-acme.nix b/ops/nixos/lib/secretsmgr-acme.nix index c85d2e8a3d..b5e6cc1598 100644 --- a/ops/nixos/lib/secretsmgr-acme.nix +++ b/ops/nixos/lib/secretsmgr-acme.nix @@ -89,5 +89,7 @@ in output_name = c.name; units_to_reload_or_restart = c.reloadOrRestartUnits; }) config.my.vault.acmeCertificates; + + users.groups = lib.listToAttrs (lib.map (groupName: lib.nameValuePair groupName {}) allGroups); }; } diff --git a/ops/nixos/rexxar/default.nix b/ops/nixos/rexxar/default.nix index 8ee565ab38..3fae753527 100644 --- a/ops/nixos/rexxar/default.nix +++ b/ops/nixos/rexxar/default.nix @@ -12,6 +12,7 @@ #../lib/nixbuild-distributed.nix # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid ../lib/hackyplayer.nix ../lib/emfminiserv.nix + ../lib/seaweedfs.nix ]; # Otherwise _this_ machine won't enumerate things properly. @@ -65,6 +66,12 @@ "/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input"; "/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output"; + "/var/lib/private/seaweedfs-master" = zfs "zu2/safe/var/lib/private/seaweedfs-master"; + "/var/lib/private/seaweedfs-filer" = zfs "zu2/safe/var/lib/private/seaweedfs-filer"; + "/var/lib/private/seaweedfs-volume" = zfs "zu2/safe/var/lib/private/seaweedfs-volume"; + "/var/lib/private/seaweedfs-volume/data" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/data"; + "/var/lib/private/seaweedfs-volume/idx" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/idx"; + "/boot" = { device = "/dev/disk/by-label/ESP"; fsType = "vfat"; @@ -409,5 +416,91 @@ enable = true; }; + my.services.seaweedfs = { + securitySettings = { + cors.allowed_origins.values = "*"; + }; + + master = { + enable = true; + options = { + port = 21000; + ip = "rexxar.int.as205479.net"; + "ip.bind" = config.my.ip.tailscale6; + mdir = "/var/lib/seaweedfs-master/metadata"; + defaultReplication = "000"; + }; + }; + filer = { + enable = true; + options = { + port = 21010; + ip = "rexxar.int.as205479.net"; + "ip.bind" = config.my.ip.tailscale6; + master = "[${config.my.ip.tailscale6}]:21000"; + encryptVolumeData = true; + + s3 = true; + "s3.domainName" = "objdump.zxcvbnm.ninja"; + "s3.localSocket" = "/run/seaweedfs-filer/s3.sock"; + "s3.port" = 21012; + + webdav = true; + "webdav.port" = 21014; + + iam = true; + "iam.ip" = "127.0.0.1"; + "iam.port" = 21015; + + defaultReplicaPlacement = "000"; + }; + settings = { + leveldb2 = { + enabled = true; + dir = "/var/lib/seaweedfs-filer/leveldb2"; + }; + }; + }; + volume = { + enable = true; + options = { + port = 21100; + ip = "rexxar.int.as205479.net"; + "ip.bind" = config.my.ip.tailscale6; + dataCenter = "ixn-lon1"; + rack = "vlx-lukegb1"; + max = 0; + mserver = "[${config.my.ip.tailscale6}]:21000"; + }; + }; + cli.settings = { + cluster.default = "rexxar"; + cluster.rexxar = { + master = "[${config.my.ip.tailscale6}]:21000"; + }; + }; + }; + my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = { + hostnames = [ + "objdump.zxcvbnm.ninja" + "*.objdump.zxcvbnm.ninja" + ]; + reloadOrRestartUnits = [ "caddy.service" ]; + }; + services.caddy = { + enable = true; + virtualHosts."objdump.zxcvbnm.ninja" = { + serverAliases = [ "*.objdump.zxcvbnm.ninja" ]; + extraConfig = '' + tls /var/lib/acme/objdump.zxcvbnm.ninja/fullchain.pem /var/lib/acme/objdump.zxcvbnm.ninja/privkey.pem + reverse_proxy unix//run/seaweedfs-filer/s3.sock + ''; + }; + }; + systemd.services.caddy.serviceConfig = { + SupplementaryGroups = lib.mkAfter [ "acme" ]; + ReadOnlyPaths = lib.mkAfter [ "/var/lib/acme" ]; + }; + system.stateVersion = "24.05"; }