Luke Granger-Brown
8b27353be7
SeaweedFS' default chunk size is 4MiB; if we don't match it (or a multiple) then our multipart uploads become uneven because they end up as chunks of the buffer size followed by the remaining data. This is particularly egregious with the current default of 5MiB, because then we get a 4MiB chunk followed by a 1MiB chunk every time.
94 lines
2.3 KiB
Go
94 lines
2.3 KiB
Go
// SPDX-FileCopyrightText: 2021 Luke Granger-Brown <depot@lukegb.com>
|
|
//
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package fuphttp
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"strconv"
|
|
"time"
|
|
|
|
"gocloud.dev/blob"
|
|
"gocloud.dev/gcerrors"
|
|
)
|
|
|
|
type Metadata struct {
|
|
// ExpiresAt is the expiry time of the object.
|
|
// May be the zero time, if this object does not expire.
|
|
// Stored in expires-at.
|
|
ExpiresAt time.Time
|
|
|
|
// Attributes is the underlying gocloud Attributes.
|
|
Attributes *blob.Attributes
|
|
}
|
|
|
|
// WriterOptions returns a new WriterOptions based on the provided metadata.
|
|
func (m *Metadata) WriterOptions() *blob.WriterOptions {
|
|
attrs := m.Attributes
|
|
if attrs == nil {
|
|
attrs = &blob.Attributes{}
|
|
}
|
|
if attrs.Metadata == nil {
|
|
attrs.Metadata = make(map[string]string)
|
|
}
|
|
|
|
if !m.ExpiresAt.IsZero() {
|
|
attrs.Metadata["expires-at"] = strconv.FormatInt(m.ExpiresAt.Unix(), 10)
|
|
}
|
|
|
|
return &blob.WriterOptions{
|
|
BufferSize: 8 * 1024 * 1024, /* 8 MiB */
|
|
CacheControl: attrs.CacheControl,
|
|
ContentDisposition: attrs.ContentDisposition,
|
|
ContentEncoding: attrs.ContentEncoding,
|
|
ContentLanguage: attrs.ContentLanguage,
|
|
ContentType: attrs.ContentType,
|
|
Metadata: attrs.Metadata,
|
|
}
|
|
}
|
|
|
|
// metadata retrieves the Metadata for the object.
|
|
// Note: if the object is expired, it will delete it.
|
|
func metadata(ctx context.Context, bucket *blob.Bucket, filename string) (*Metadata, error) {
|
|
attrs, err := bucket.Attributes(ctx, filename)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Attributes(%q): %w", filename, err)
|
|
}
|
|
|
|
m := Metadata{
|
|
Attributes: attrs,
|
|
}
|
|
|
|
var exp time.Time
|
|
if expStr, ok := attrs.Metadata["expires-at"]; ok {
|
|
// Check for expiry.
|
|
expSec, err := strconv.ParseInt(expStr, 10, 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing expiration %q for %q: %v", expStr, filename, err)
|
|
}
|
|
|
|
exp = time.Unix(expSec, 0)
|
|
if exp.Before(time.Now()) {
|
|
// This file has expired. Delete it from the backing storage.
|
|
err := bucket.Delete(ctx, filename)
|
|
switch errorCode(err) {
|
|
case gcerrors.NotFound:
|
|
// Something probably deleted it before we could get there.
|
|
case gcerrors.OK:
|
|
// This was fine.
|
|
default:
|
|
log.Printf("deleting expired file %q: %v", filename, err)
|
|
}
|
|
return nil, &fupError{
|
|
Code: gcerrors.NotFound,
|
|
}
|
|
}
|
|
|
|
m.ExpiresAt = exp
|
|
}
|
|
|
|
return &m, nil
|
|
}
|