ops/nixos: introduce seaweedfs

This commit is contained in:
Luke Granger-Brown 2024-10-20 14:37:47 +01:00
parent df552bf3b4
commit 525ec8977b
4 changed files with 304 additions and 4 deletions

View file

@ -340,9 +340,6 @@
allow_websockets = true; allow_websockets = true;
timeout = "0"; timeout = "0";
}) })
(service "blade-tuvok.int.as205479.net:7480" "objdump.zxcvbnm.ninja" (public {
timeout = "30m"; # Uploads can take a while; bump the timeout.
}))
(secureService "totoro.int.as205479.net" "invoices.lukegb.com" (public { (secureService "totoro.int.as205479.net" "invoices.lukegb.com" (public {
regex = "^/((third_party|ajax|client_area|pdf)/.*|[a-zA-Z0-9]{8})$"; regex = "^/((third_party|ajax|client_area|pdf)/.*|[a-zA-Z0-9]{8})$";
tls_skip_verify = true; tls_skip_verify = true;
@ -403,7 +400,6 @@
"lukegb.com" "lukegb.com"
"*.lukegb.com" "*.lukegb.com"
"*.int.lukegb.com" "*.int.lukegb.com"
"objdump.zxcvbnm.ninja"
]; ];
reloadOrRestartUnits = [ "pomerium.service" ]; reloadOrRestartUnits = [ "pomerium.service" ];
}; };

209
ops/nixos/lib/seaweedfs.nix Normal file
View file

@ -0,0 +1,209 @@
{ depot, config, pkgs, lib, ... }:
let
cfg = config.my.services.seaweedfs;
tomlFormat = pkgs.formats.toml {};
makeCommandLine' = prefix: options: let
makeFlag = name: value:
if builtins.isAttrs value then makeCommandLine' "${prefix}${name}." value
else if builtins.isString value then "-${prefix}${name}=${lib.strings.escapeShellArg value}"
else if builtins.isInt value then "-${prefix}${name}=${toString value}"
else "-${name}";
optionsList = lib.attrsets.mapAttrsToList makeFlag options;
in
lib.strings.concatStringsSep " " optionsList;
makeCommandLine = makeCommandLine' "";
commandLineType = lib.types.oneOf [
lib.types.bool
lib.types.int
lib.types.str
(lib.types.attrsOf commandLineType)
];
in {
options.my.services.seaweedfs = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.seaweedfs;
};
securitySettings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
filer = {
enable = lib.mkEnableOption "SeaweedFS filer";
package = lib.mkOption {
type = lib.types.package;
};
options = lib.mkOption {
type = lib.types.submodule {
freeformType = commandLineType;
};
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
notificationSettings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
replicationSettings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
};
master = {
enable = lib.mkEnableOption "SeaweedFS master";
package = lib.mkOption {
type = lib.types.package;
};
options = lib.mkOption {
type = lib.types.submodule {
freeformType = commandLineType;
};
default = {};
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
};
volume = {
enable = lib.mkEnableOption "SeaweedFS volume";
package = lib.mkOption {
type = lib.types.package;
};
options = lib.mkOption {
type = lib.types.submodule {
freeformType = commandLineType;
};
default = {};
};
};
cli = {
enable = lib.mkEnableOption "add SeaweedFS CLI";
package = lib.mkOption {
type = lib.types.package;
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = tomlFormat.type;
};
default = {};
};
};
};
config = lib.mkMerge [{
my.services.seaweedfs.master.package = lib.mkDefault cfg.package;
my.services.seaweedfs.filer.package = lib.mkDefault cfg.package;
my.services.seaweedfs.volume.package = lib.mkDefault cfg.package;
my.services.seaweedfs.cli.package = lib.mkDefault cfg.package;
} (lib.mkIf (cfg.filer.enable || cfg.master.enable || cfg.volume.enable) {
environment.etc."seaweedfs/security.toml".source = tomlFormat.generate "seaweedfs-security.toml" cfg.securitySettings;
my.services.seaweedfs.cli.enable = lib.mkDefault true;
users.groups.seaweedfs = {};
systemd.targets.seaweedfs = {
description = "SeaweedFS components";
wantedBy = [ "multi-user.target" ];
};
}) (lib.mkIf (cfg.cli.enable) {
environment.systemPackages = [ cfg.cli.package ];
environment.etc."seaweedfs/shell.toml".source = tomlFormat.generate "seaweedfs-shell.toml" cfg.cli.settings;
}) (lib.mkIf (cfg.filer.enable) {
environment.etc."seaweedfs/filer.toml".source = tomlFormat.generate "seaweedfs-filer.toml" cfg.filer.settings;
environment.etc."seaweedfs/notification.toml".source = tomlFormat.generate "seaweedfs-notification.toml" cfg.filer.notificationSettings;
environment.etc."seaweedfs/replication.toml".source = tomlFormat.generate "seaweedfs-replication.toml" cfg.filer.replicationSettings;
my.services.seaweedfs.filer.options = {
localSocket = lib.mkDefault "/run/seaweedfs-filer/filer.sock";
};
systemd.services."seaweedfs-filer" = {
wantedBy = [ "seaweedfs.target" ];
restartTriggers = [
(config.environment.etc."seaweedfs/security.toml".source)
(config.environment.etc."seaweedfs/filer.toml".source)
(config.environment.etc."seaweedfs/notification.toml".source)
(config.environment.etc."seaweedfs/replication.toml".source)
];
serviceConfig = {
ExecStart = "${cfg.filer.package}/bin/weed filer ${makeCommandLine cfg.filer.options}";
User = "seaweedfs-filer";
Group = "seaweedfs";
DynamicUser = true;
RuntimeDirectory = "seaweedfs-filer";
StateDirectory = "seaweedfs-filer";
};
};
}) (lib.mkIf (cfg.master.enable) {
environment.etc."seaweedfs/master.toml".source = tomlFormat.generate "seaweedfs-master.toml" cfg.master.settings;
systemd.services."seaweedfs-master" = {
wantedBy = [ "seaweedfs.target" ];
restartTriggers = [
(config.environment.etc."seaweedfs/security.toml".source)
];
serviceConfig = {
ExecStart = "${cfg.master.package}/bin/weed master ${makeCommandLine cfg.master.options}";
User = "seaweedfs-master";
Group = "seaweedfs";
DynamicUser = true;
RuntimeDirectory = "seaweedfs-master";
StateDirectory = "seaweedfs-master";
};
};
}) (lib.mkIf (cfg.volume.enable) {
my.services.seaweedfs.volume.options = {
dir = lib.mkDefault "/var/lib/seaweedfs-volume/data";
"dir.idx" = lib.mkDefault "/var/lib/seaweedfs-volume/idx";
};
systemd.services."seaweedfs-volume" = {
wantedBy = [ "seaweedfs.target" ];
restartTriggers = [
(config.environment.etc."seaweedfs/security.toml".source)
];
serviceConfig = {
ExecStart = "${cfg.volume.package}/bin/weed volume ${makeCommandLine cfg.volume.options}";
User = "seaweedfs-volume";
Group = "seaweedfs";
DynamicUser = true;
RuntimeDirectory = "seaweedfs-volume";
StateDirectory = "seaweedfs-volume";
};
};
})];
}

View file

@ -89,5 +89,7 @@ in
output_name = c.name; output_name = c.name;
units_to_reload_or_restart = c.reloadOrRestartUnits; units_to_reload_or_restart = c.reloadOrRestartUnits;
}) config.my.vault.acmeCertificates; }) config.my.vault.acmeCertificates;
users.groups = lib.listToAttrs (lib.map (groupName: lib.nameValuePair groupName {}) allGroups);
}; };
} }

View file

@ -12,6 +12,7 @@
#../lib/nixbuild-distributed.nix # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid #../lib/nixbuild-distributed.nix # error: build of '/nix/store/3r7456yr8r9g4fl7w6xbgqlbsdjwfvr4-stdlib-pkgs.json.drv' on 'ssh://eu.nixbuild.net' failed: unexpected: Built outputs are invalid
../lib/hackyplayer.nix ../lib/hackyplayer.nix
../lib/emfminiserv.nix ../lib/emfminiserv.nix
../lib/seaweedfs.nix
]; ];
# Otherwise _this_ machine won't enumerate things properly. # Otherwise _this_ machine won't enumerate things properly.
@ -65,6 +66,12 @@
"/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input"; "/store/emf/2024/video/input" = zfs "zu2/safe/store/emf/2024/video/input";
"/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output"; "/store/emf/2024/video/output" = zfs "zu2/safe/store/emf/2024/video/output";
"/var/lib/private/seaweedfs-master" = zfs "zu2/safe/var/lib/private/seaweedfs-master";
"/var/lib/private/seaweedfs-filer" = zfs "zu2/safe/var/lib/private/seaweedfs-filer";
"/var/lib/private/seaweedfs-volume" = zfs "zu2/safe/var/lib/private/seaweedfs-volume";
"/var/lib/private/seaweedfs-volume/data" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/data";
"/var/lib/private/seaweedfs-volume/idx" = zfs "zu2/safe/var/lib/private/seaweedfs-volume/idx";
"/boot" = { "/boot" = {
device = "/dev/disk/by-label/ESP"; device = "/dev/disk/by-label/ESP";
fsType = "vfat"; fsType = "vfat";
@ -409,5 +416,91 @@
enable = true; enable = true;
}; };
my.services.seaweedfs = {
securitySettings = {
cors.allowed_origins.values = "*";
};
master = {
enable = true;
options = {
port = 21000;
ip = "rexxar.int.as205479.net";
"ip.bind" = config.my.ip.tailscale6;
mdir = "/var/lib/seaweedfs-master/metadata";
defaultReplication = "000";
};
};
filer = {
enable = true;
options = {
port = 21010;
ip = "rexxar.int.as205479.net";
"ip.bind" = config.my.ip.tailscale6;
master = "[${config.my.ip.tailscale6}]:21000";
encryptVolumeData = true;
s3 = true;
"s3.domainName" = "objdump.zxcvbnm.ninja";
"s3.localSocket" = "/run/seaweedfs-filer/s3.sock";
"s3.port" = 21012;
webdav = true;
"webdav.port" = 21014;
iam = true;
"iam.ip" = "127.0.0.1";
"iam.port" = 21015;
defaultReplicaPlacement = "000";
};
settings = {
leveldb2 = {
enabled = true;
dir = "/var/lib/seaweedfs-filer/leveldb2";
};
};
};
volume = {
enable = true;
options = {
port = 21100;
ip = "rexxar.int.as205479.net";
"ip.bind" = config.my.ip.tailscale6;
dataCenter = "ixn-lon1";
rack = "vlx-lukegb1";
max = 0;
mserver = "[${config.my.ip.tailscale6}]:21000";
};
};
cli.settings = {
cluster.default = "rexxar";
cluster.rexxar = {
master = "[${config.my.ip.tailscale6}]:21000";
};
};
};
my.vault.acmeCertificates."objdump.zxcvbnm.ninja" = {
hostnames = [
"objdump.zxcvbnm.ninja"
"*.objdump.zxcvbnm.ninja"
];
reloadOrRestartUnits = [ "caddy.service" ];
};
services.caddy = {
enable = true;
virtualHosts."objdump.zxcvbnm.ninja" = {
serverAliases = [ "*.objdump.zxcvbnm.ninja" ];
extraConfig = ''
tls /var/lib/acme/objdump.zxcvbnm.ninja/fullchain.pem /var/lib/acme/objdump.zxcvbnm.ninja/privkey.pem
reverse_proxy unix//run/seaweedfs-filer/s3.sock
'';
};
};
systemd.services.caddy.serviceConfig = {
SupplementaryGroups = lib.mkAfter [ "acme" ];
ReadOnlyPaths = lib.mkAfter [ "/var/lib/acme" ];
};
system.stateVersion = "24.05"; system.stateVersion = "24.05";
} }