From a021fff1da6d356bbeeef5b582e34c6ffb8b4d8e Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 20 Oct 2024 17:21:25 +0100 Subject: [PATCH] ops/nixos/seaweedfs: switch to locally patched seaweedfs --- nix/pkgs/default.nix | 2 + ...ressed-chunks-when-finalizing-multip.patch | 49 +++++++ ...a.edit-for-editing-a-filer-metadata-.patch | 134 ++++++++++++++++++ nix/pkgs/seaweedfs/default.nix | 8 ++ ops/nixos/lib/seaweedfs.nix | 2 +- 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 nix/pkgs/seaweedfs/0001-s3-remember-compressed-chunks-when-finalizing-multip.patch create mode 100644 nix/pkgs/seaweedfs/0002-shell-add-fs.meta.edit-for-editing-a-filer-metadata-.patch create mode 100644 nix/pkgs/seaweedfs/default.nix diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 2f9d94fbb1..2be7541e95 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -89,4 +89,6 @@ open5gs = pkgs.callPackage ./open5gs { }; zigbee2mqtt = pkgs.callPackage ./zigbee2mqtt.nix { }; + + seaweedfs = pkgs.callPackage ./seaweedfs { }; } // (import ./heptapod-runner args) diff --git a/nix/pkgs/seaweedfs/0001-s3-remember-compressed-chunks-when-finalizing-multip.patch b/nix/pkgs/seaweedfs/0001-s3-remember-compressed-chunks-when-finalizing-multip.patch new file mode 100644 index 0000000000..bccd3d134f --- /dev/null +++ b/nix/pkgs/seaweedfs/0001-s3-remember-compressed-chunks-when-finalizing-multip.patch @@ -0,0 +1,49 @@ +From 624e2d79ce5a03bb60afd8e1293089dd61c60d52 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown +Date: Sun, 20 Oct 2024 16:12:10 +0100 +Subject: [PATCH 1/2] [s3] remember compressed chunks when finalizing multipart + uploads + +Any compressed chunks will forget that they're compressed when the +multipart upload is finalized. This breaks content readback if e.g. the +chunks are encrypted, because the sizes won't match the expectation. +--- + weed/s3api/filer_multipart.go | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go +index 43f982897..76d27f460 100644 +--- a/weed/s3api/filer_multipart.go ++++ b/weed/s3api/filer_multipart.go +@@ -5,9 +5,6 @@ import ( + "encoding/hex" + "encoding/xml" + "fmt" +- "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +- "github.com/seaweedfs/seaweedfs/weed/stats" +- "golang.org/x/exp/slices" + "math" + "path/filepath" + "sort" +@@ -15,6 +12,10 @@ import ( + "strings" + "time" + ++ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" ++ "github.com/seaweedfs/seaweedfs/weed/stats" ++ "golang.org/x/exp/slices" ++ + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/google/uuid" +@@ -209,6 +210,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa + ModifiedTsNs: chunk.ModifiedTsNs, + CipherKey: chunk.CipherKey, + ETag: chunk.ETag, ++ IsCompressed: chunk.IsCompressed, + } + finalParts = append(finalParts, p) + offset += int64(chunk.Size) +-- +2.46.0 + diff --git a/nix/pkgs/seaweedfs/0002-shell-add-fs.meta.edit-for-editing-a-filer-metadata-.patch b/nix/pkgs/seaweedfs/0002-shell-add-fs.meta.edit-for-editing-a-filer-metadata-.patch new file mode 100644 index 0000000000..6dee2f4e1e --- /dev/null +++ b/nix/pkgs/seaweedfs/0002-shell-add-fs.meta.edit-for-editing-a-filer-metadata-.patch @@ -0,0 +1,134 @@ +From 74c486e9f8b2557e4bf764adbbd46642c175ea40 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown +Date: Sun, 20 Oct 2024 16:59:21 +0100 +Subject: [PATCH 2/2] [shell] add fs.meta.edit for editing a filer metadata + entry + +--- + weed/shell/command_fs_meta_edit.go | 114 +++++++++++++++++++++++++++++ + 1 file changed, 114 insertions(+) + create mode 100644 weed/shell/command_fs_meta_edit.go + +diff --git a/weed/shell/command_fs_meta_edit.go b/weed/shell/command_fs_meta_edit.go +new file mode 100644 +index 000000000..ece203b4d +--- /dev/null ++++ b/weed/shell/command_fs_meta_edit.go +@@ -0,0 +1,114 @@ ++package shell ++ ++import ( ++ "flag" ++ "fmt" ++ "io" ++ "os" ++ "os/exec" ++ ++ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ++ "github.com/seaweedfs/seaweedfs/weed/util" ++ "google.golang.org/protobuf/encoding/prototext" ++) ++ ++func init() { ++ Commands = append(Commands, &commandFsMetaEdit{}) ++} ++ ++type commandFsMetaEdit struct { ++ dirPrefix *string ++} ++ ++func (c *commandFsMetaEdit) Name() string { ++ return "fs.meta.edit" ++} ++ ++func (c *commandFsMetaEdit) Help() string { ++ return `edit single file's metadata ++ ++ fs.meta.edit /path/to/single/file ++` ++} ++ ++func (c *commandFsMetaEdit) HasTag(CommandTag) bool { ++ return false ++} ++ ++func (c *commandFsMetaEdit) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { ++ ++ if len(args) == 0 { ++ fmt.Fprintf(writer, "missing a path to a file\n") ++ return nil ++ } ++ ++ metaEditCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) ++ if err = metaEditCommand.Parse(args); err != nil { ++ return nil ++ } ++ path, parseErr := commandEnv.parseUrl(findInputDirectory(metaEditCommand.Args())) ++ if parseErr != nil { ++ return parseErr ++ } ++ filerPath := util.FullPath(path) ++ ++ // Fetch entry. ++ fmt.Fprintf(writer, "%#v\n", filerPath) ++ entry, err := filer_pb.GetEntry(commandEnv, filerPath) ++ if err != nil { ++ return fmt.Errorf("getting entry data: %w", err) ++ } ++ ++ // Write entry to temp file. ++ f, err := os.CreateTemp("", "seaweed-entry-*.txtpb") ++ if err != nil { ++ return fmt.Errorf("creating temp file: %w", err) ++ } ++ defer os.Remove(f.Name()) ++ ++ if _, err := f.Write([]byte(prototext.Format(entry))); err != nil { ++ return fmt.Errorf("writing formatted entry data: %w", err) ++ } ++ ++ // Launch an editor. ++ editor := os.Getenv("VISUAL") ++ if editor == "" { ++ editor = os.Getenv("EDITOR") ++ } ++ if editor == "" { ++ editor = "vim" ++ } ++ cmd := exec.Command("sh", "-c", fmt.Sprintf("%s %s", editor, f.Name())) ++ cmd.Stdin = os.Stdin ++ cmd.Stdout = os.Stdout ++ cmd.Stderr = os.Stderr ++ if err := cmd.Run(); err != nil { ++ return fmt.Errorf("editor failed: %w", err) ++ } ++ ++ // Read edited entry. ++ if _, err := f.Seek(0, 0); err != nil { ++ return fmt.Errorf("seeking to beginning of file failed: %w", err) ++ } ++ editedBytes, err := io.ReadAll(f) ++ if err != nil { ++ return fmt.Errorf("reading edited entry: %w", err) ++ } ++ if err := prototext.Unmarshal(editedBytes, entry); err != nil { ++ return fmt.Errorf("parsing edited entry: %w", err) ++ } ++ ++ // Write back entry to filer. ++ dir, _ := filerPath.DirAndName() ++ req := &filer_pb.UpdateEntryRequest{ ++ Directory: dir, ++ Entry: entry, ++ } ++ if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { ++ return filer_pb.UpdateEntry(client, req) ++ }); err != nil { ++ return fmt.Errorf("updating entry in filer: %w", err) ++ } ++ ++ return nil ++} +-- +2.46.0 + diff --git a/nix/pkgs/seaweedfs/default.nix b/nix/pkgs/seaweedfs/default.nix new file mode 100644 index 0000000000..e6d64947f4 --- /dev/null +++ b/nix/pkgs/seaweedfs/default.nix @@ -0,0 +1,8 @@ +{ seaweedfs }: + +seaweedfs.overrideAttrs (_: _: { + patches = [ + ./0001-s3-remember-compressed-chunks-when-finalizing-multip.patch + ./0002-shell-add-fs.meta.edit-for-editing-a-filer-metadata-.patch + ]; +}) diff --git a/ops/nixos/lib/seaweedfs.nix b/ops/nixos/lib/seaweedfs.nix index aec63a53c3..b7a9991252 100644 --- a/ops/nixos/lib/seaweedfs.nix +++ b/ops/nixos/lib/seaweedfs.nix @@ -26,7 +26,7 @@ in { options.my.services.seaweedfs = { package = lib.mkOption { type = lib.types.package; - default = pkgs.seaweedfs; + default = depot.nix.pkgs.seaweedfs; # pkgs.seaweedfs; }; securitySettings = lib.mkOption {