From 525ec8977b7864355766686e2360932b0f4e155c Mon Sep 17 00:00:00 2001
From: Luke Granger-Brown <hg@lukegb.com>
Date: Sun, 20 Oct 2024 14:37:47 +0100
Subject: [PATCH] ops/nixos: introduce seaweedfs

---
 ops/nixos/etheroute-lon01/default.nix |   4 -
 ops/nixos/lib/seaweedfs.nix           | 209 ++++++++++++++++++++++++++
 ops/nixos/lib/secretsmgr-acme.nix     |   2 +
 ops/nixos/rexxar/default.nix          |  93 ++++++++++++
 4 files changed, 304 insertions(+), 4 deletions(-)
 create mode 100644 ops/nixos/lib/seaweedfs.nix

diff --git a/ops/nixos/etheroute-lon01/default.nix b/ops/nixos/etheroute-lon01/default.nix
index c8ee245911..06d7a80443 100644
--- a/ops/nixos/etheroute-lon01/default.nix
+++ b/ops/nixos/etheroute-lon01/default.nix
@@ -340,9 +340,6 @@
           allow_websockets = true;
           timeout = "0";
         })
-        (service "blade-tuvok.int.as205479.net:7480" "objdump.zxcvbnm.ninja" (public {
-          timeout = "30m";  # Uploads can take a while; bump the timeout.
-        }))
         (secureService "totoro.int.as205479.net" "invoices.lukegb.com" (public {
           regex = "^/((third_party|ajax|client_area|pdf)/.*|[a-zA-Z0-9]{8})$";
           tls_skip_verify = true;
@@ -403,7 +400,6 @@
       "lukegb.com"
       "*.lukegb.com"
       "*.int.lukegb.com"
-      "objdump.zxcvbnm.ninja"
     ];
     reloadOrRestartUnits = [ "pomerium.service" ];
   };
diff --git a/ops/nixos/lib/seaweedfs.nix b/ops/nixos/lib/seaweedfs.nix
new file mode 100644
index 0000000000..aec63a53c3
--- /dev/null
+++ b/ops/nixos/lib/seaweedfs.nix
@@ -0,0 +1,209 @@
+{ depot, config, pkgs, lib, ... }:
+
+let
+  cfg = config.my.services.seaweedfs;
+
+  tomlFormat = pkgs.formats.toml {};
+
+  makeCommandLine' = prefix: options: let
+    makeFlag = name: value:
+      if builtins.isAttrs value then makeCommandLine' "${prefix}${name}." value
+      else if builtins.isString value then "-${prefix}${name}=${lib.strings.escapeShellArg value}"
+      else if builtins.isInt value then "-${prefix}${name}=${toString value}"
+      else "-${name}";
+    optionsList = lib.attrsets.mapAttrsToList makeFlag options;
+  in
+    lib.strings.concatStringsSep " " optionsList;
+  makeCommandLine = makeCommandLine' "";
+
+  commandLineType = lib.types.oneOf [
+    lib.types.bool
+    lib.types.int
+    lib.types.str
+    (lib.types.attrsOf commandLineType)
+  ];
+in {
+  options.my.services.seaweedfs = {
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.seaweedfs;
+    };
+
+    securitySettings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = tomlFormat.type;
+      };
+      default = {};
+    };
+
+    filer = {
+      enable = lib.mkEnableOption "SeaweedFS filer";
+
+      package = lib.mkOption {
+        type = lib.types.package;
+      };
+
+      options = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = commandLineType;
+        };
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = tomlFormat.type;
+        };
+        default = {};
+      };
+      notificationSettings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = tomlFormat.type;
+        };
+        default = {};
+      };
+      replicationSettings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = tomlFormat.type;
+        };
+        default = {};
+      };
+    };
+
+    master = {
+      enable = lib.mkEnableOption "SeaweedFS master";
+
+      package = lib.mkOption {
+        type = lib.types.package;
+      };
+
+      options = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = commandLineType;
+        };
+        default = {};
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = tomlFormat.type;
+        };
+        default = {};
+      };
+    };
+
+    volume = {
+      enable = lib.mkEnableOption "SeaweedFS volume";
+
+      package = lib.mkOption {
+        type = lib.types.package;
+      };
+
+      options = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = commandLineType;
+        };
+        default = {};
+      };
+    };
+
+    cli = {
+      enable = lib.mkEnableOption "add SeaweedFS CLI";
+
+      package = lib.mkOption {
+        type = lib.types.package;
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = tomlFormat.type;
+        };
+        default = {};
+      };
+    };
+  };
+
+  config = lib.mkMerge [{
+    my.services.seaweedfs.master.package = lib.mkDefault cfg.package;
+    my.services.seaweedfs.filer.package = lib.mkDefault cfg.package;
+    my.services.seaweedfs.volume.package = lib.mkDefault cfg.package;
+    my.services.seaweedfs.cli.package = lib.mkDefault cfg.package;
+  } (lib.mkIf (cfg.filer.enable || cfg.master.enable || cfg.volume.enable) {
+    environment.etc."seaweedfs/security.toml".source = tomlFormat.generate "seaweedfs-security.toml" cfg.securitySettings;
+
+    my.services.seaweedfs.cli.enable = lib.mkDefault true;
+
+    users.groups.seaweedfs = {};
+
+    systemd.targets.seaweedfs = {
+      description = "SeaweedFS components";
+      wantedBy = [ "multi-user.target" ];
+    };
+  }) (lib.mkIf (cfg.cli.enable) {
+    environment.systemPackages = [ cfg.cli.package ];
+
+    environment.etc."seaweedfs/shell.toml".source = tomlFormat.generate "seaweedfs-shell.toml" cfg.cli.settings;
+  }) (lib.mkIf (cfg.filer.enable) {
+    environment.etc."seaweedfs/filer.toml".source = tomlFormat.generate "seaweedfs-filer.toml" cfg.filer.settings;
+    environment.etc."seaweedfs/notification.toml".source = tomlFormat.generate "seaweedfs-notification.toml" cfg.filer.notificationSettings;
+    environment.etc."seaweedfs/replication.toml".source = tomlFormat.generate "seaweedfs-replication.toml" cfg.filer.replicationSettings;
+
+    my.services.seaweedfs.filer.options = {
+      localSocket = lib.mkDefault "/run/seaweedfs-filer/filer.sock";
+    };
+
+    systemd.services."seaweedfs-filer" = {
+      wantedBy = [ "seaweedfs.target" ];
+      restartTriggers = [
+        (config.environment.etc."seaweedfs/security.toml".source)
+        (config.environment.etc."seaweedfs/filer.toml".source)
+        (config.environment.etc."seaweedfs/notification.toml".source)
+        (config.environment.etc."seaweedfs/replication.toml".source)
+      ];
+      serviceConfig = {
+        ExecStart = "${cfg.filer.package}/bin/weed filer ${makeCommandLine cfg.filer.options}";
+        User = "seaweedfs-filer";
+        Group = "seaweedfs";
+        DynamicUser = true;
+        RuntimeDirectory = "seaweedfs-filer";
+        StateDirectory = "seaweedfs-filer";
+      };
+    };
+  }) (lib.mkIf (cfg.master.enable) {
+    environment.etc."seaweedfs/master.toml".source = tomlFormat.generate "seaweedfs-master.toml" cfg.master.settings;
+
+    systemd.services."seaweedfs-master" = {
+      wantedBy = [ "seaweedfs.target" ];
+      restartTriggers = [
+        (config.environment.etc."seaweedfs/security.toml".source)
+      ];
+      serviceConfig = {
+        ExecStart = "${cfg.master.package}/bin/weed master ${makeCommandLine cfg.master.options}";
+        User = "seaweedfs-master";
+        Group = "seaweedfs";
+        DynamicUser = true;
+        RuntimeDirectory = "seaweedfs-master";
+        StateDirectory = "seaweedfs-master";
+      };
+    };
+  }) (lib.mkIf (cfg.volume.enable) {
+    my.services.seaweedfs.volume.options = {
+      dir = lib.mkDefault "/var/lib/seaweedfs-volume/data";
+      "dir.idx" = lib.mkDefault "/var/lib/seaweedfs-volume/idx";
+    };
+
+    systemd.services."seaweedfs-volume" = {
+      wantedBy = [ "seaweedfs.target" ];
+      restartTriggers = [
+        (config.environment.etc."seaweedfs/security.toml".source)
+      ];
+      serviceConfig = {
+        ExecStart = "${cfg.volume.package}/bin/weed volume ${makeCommandLine cfg.volume.options}";
+        User = "seaweedfs-volume";
+        Group = "seaweedfs";
+        DynamicUser = true;
+        RuntimeDirectory = "seaweedfs-volume";
+        StateDirectory = "seaweedfs-volume";
+      };
+    };
+  })];
+}
diff --git a/ops/nixos/lib/secretsmgr-acme.nix b/ops/nixos/lib/secretsmgr-acme.nix
index c85d2e8a3d..b5e6cc1598 100644
--- a/ops/nixos/lib/secretsmgr-acme.nix
+++ b/ops/nixos/lib/secretsmgr-acme.nix
@@ -89,5 +89,7 @@ in
       output_name = c.name;
       units_to_reload_or_restart = c.reloadOrRestartUnits;
     }) config.my.vault.acmeCertificates;
+
+    users.groups = lib.listToAttrs (lib.map (groupName: lib.nameValuePair groupName {}) allGroups);
   };
 }
diff --git a/ops/nixos/rexxar/default.nix b/ops/nixos/rexxar/default.nix
index 8ee565ab38..3fae753527 100644
--- a/ops/nixos/rexxar/default.nix
+++ b/ops/nixos/rexxar/default.nix
@@ -12,6 +12,7 @@
     #../lib/nixbuild-distributed.nix  # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid
     ../lib/hackyplayer.nix
     ../lib/emfminiserv.nix
+    ../lib/seaweedfs.nix
   ];
 
   # Otherwise _this_ machine won't enumerate things properly.
@@ -65,6 +66,12 @@
     "/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input";
     "/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output";
 
+    "/var/lib/private/seaweedfs-master" = zfs "zu2/safe/var/lib/private/seaweedfs-master";
+    "/var/lib/private/seaweedfs-filer" = zfs "zu2/safe/var/lib/private/seaweedfs-filer";
+    "/var/lib/private/seaweedfs-volume" = zfs "zu2/safe/var/lib/private/seaweedfs-volume";
+    "/var/lib/private/seaweedfs-volume/data" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/data";
+    "/var/lib/private/seaweedfs-volume/idx" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/idx";
+
     "/boot" = {
       device = "/dev/disk/by-label/ESP";
       fsType = "vfat";
@@ -409,5 +416,91 @@
     enable = true;
   };
 
+  my.services.seaweedfs = {
+    securitySettings = {
+      cors.allowed_origins.values = "*";
+    };
+
+    master = {
+      enable = true;
+      options = {
+        port = 21000;
+        ip = "rexxar.int.as205479.net";
+        "ip.bind" = config.my.ip.tailscale6;
+        mdir = "/var/lib/seaweedfs-master/metadata";
+        defaultReplication = "000";
+      };
+    };
+    filer = {
+      enable = true;
+      options = {
+        port = 21010;
+        ip = "rexxar.int.as205479.net";
+        "ip.bind" = config.my.ip.tailscale6;
+        master = "[${config.my.ip.tailscale6}]:21000";
+        encryptVolumeData = true;
+
+        s3 = true;
+        "s3.domainName" = "objdump.zxcvbnm.ninja";
+        "s3.localSocket" = "/run/seaweedfs-filer/s3.sock";
+        "s3.port" = 21012;
+
+        webdav = true;
+        "webdav.port" = 21014;
+
+        iam = true;
+        "iam.ip" = "127.0.0.1";
+        "iam.port" = 21015;
+
+        defaultReplicaPlacement = "000";
+      };
+      settings = {
+        leveldb2 = {
+          enabled = true;
+          dir = "/var/lib/seaweedfs-filer/leveldb2";
+        };
+      };
+    };
+    volume = {
+      enable = true;
+      options = {
+        port = 21100;
+        ip = "rexxar.int.as205479.net";
+        "ip.bind" = config.my.ip.tailscale6;
+        dataCenter = "ixn-lon1";
+        rack = "vlx-lukegb1";
+        max = 0;
+        mserver = "[${config.my.ip.tailscale6}]:21000";
+      };
+    };
+    cli.settings = {
+      cluster.default = "rexxar";
+      cluster.rexxar = {
+        master = "[${config.my.ip.tailscale6}]:21000";
+      };
+    };
+  };
+  my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = {
+    hostnames = [
+      "objdump.zxcvbnm.ninja"
+      "*.objdump.zxcvbnm.ninja"
+    ];
+    reloadOrRestartUnits = [ "caddy.service" ];
+  };
+  services.caddy = {
+    enable = true;
+    virtualHosts."objdump.zxcvbnm.ninja" = {
+      serverAliases = [ "*.objdump.zxcvbnm.ninja" ];
+      extraConfig = ''
+        tls /var/lib/acme/objdump.zxcvbnm.ninja/fullchain.pem /var/lib/acme/objdump.zxcvbnm.ninja/privkey.pem
+        reverse_proxy unix//run/seaweedfs-filer/s3.sock
+      '';
+    };
+  };
+  systemd.services.caddy.serviceConfig = {
+    SupplementaryGroups = lib.mkAfter [ "acme" ];
+    ReadOnlyPaths = lib.mkAfter [ "/var/lib/acme" ];
+  };
+
   system.stateVersion = "24.05";
 }