From 08098fb6665fabf2c4c91ef779326a0cc8a8136a Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 21 Mar 2021 03:04:38 +0000 Subject: [PATCH] fup: add file serving This adds both redirect-to-signed-URL and proxy fileserving. The proxy fileserving is somewhat limited: we don't support the Range header, and it isn't easy to reuse the net/http ServeContent implementation because that requires a SeekCloser. I think it might be possible to "bodge" a SeekCloser on top of dynamically opening files, but it'll be a bit wonky and will be slower than strictly necessary. --- web/fup/cmd/serve.go | 15 +++- web/fup/fuphttp/fuphttp.go | 159 +++++++++++++++++++++++++++++++++++-- 2 files changed, 163 insertions(+), 11 deletions(-) diff --git a/web/fup/cmd/serve.go b/web/fup/cmd/serve.go index 8c06ee6d4f..1bc4ef5f95 100644 --- a/web/fup/cmd/serve.go +++ b/web/fup/cmd/serve.go @@ -7,8 +7,10 @@ package cmd import ( "context" "fmt" + "log" "net/http" + "github.com/google/safehtml" "github.com/spf13/cobra" "hg.lukegb.com/lukegb/depot/web/fup/fuphttp" "hg.lukegb.com/lukegb/depot/web/fup/fupstatic" @@ -18,10 +20,12 @@ func init() { rootCmd.AddCommand(serveCmd) serveCmd.Flags().StringVarP(&serveBind, "listen", "l", ":8191", "Bind address for HTTP server.") + serveCmd.Flags().BoolVar(&serveDirectOnly, "direct-only", false, "If set, all file serving will be proxied, even if the backend supports signed URLs.") } var ( - serveBind string + serveBind string + serveDirectOnly bool serveCmd = &cobra.Command{ Use: "serve", @@ -29,9 +33,11 @@ var ( RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() cfg := &fuphttp.Config{ - Templates: fupstatic.Templates, - Static: fupstatic.Static, - StaticRoot: "/static/", + Templates: fupstatic.Templates, + Static: fupstatic.Static, + StaticRoot: safehtml.TrustedResourceURLFromConstant("/static/"), + StorageURL: bucketURL, + RedirectToBlobstore: !serveDirectOnly, } a, err := fuphttp.New(ctx, cfg) if err != nil { @@ -39,6 +45,7 @@ var ( } http.Handle("/", a.Handler()) http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.FS(fupstatic.Static)))) + log.Printf("Serving on %s", serveBind) return http.ListenAndServe(serveBind, nil) }, } diff --git a/web/fup/fuphttp/fuphttp.go b/web/fup/fuphttp/fuphttp.go index ab1a5dacf3..39841584d1 100644 --- a/web/fup/fuphttp/fuphttp.go +++ b/web/fup/fuphttp/fuphttp.go @@ -7,39 +7,169 @@ package fuphttp import ( "context" "fmt" + "io" "io/fs" + "log" "net/http" + "time" "github.com/google/safehtml" "github.com/google/safehtml/template" "github.com/google/safehtml/template/uncheckedconversions" shuncheckedconversions "github.com/google/safehtml/uncheckedconversions" "github.com/gorilla/mux" + "gocloud.dev/blob" + "gocloud.dev/gcerrors" "hg.lukegb.com/lukegb/depot/web/fup/hashfs" ) +const ( + defaultRedirectExpiry = 5 * time.Minute +) + type Config struct { Templates fs.FS Static fs.FS StaticRoot safehtml.TrustedResourceURL + + // If set, redirects to a signed URL if possible instead of serving directly. + RedirectToBlobstore bool + RedirectExpiry time.Duration // Defaults to 5 minutes. + + // Set one of these, but not both. + StorageURL string + StorageBackend *blob.Bucket } type Application struct { - indexTmpl *template.Template - notFoundTmpl *template.Template + indexTmpl *template.Template + notFoundTmpl *template.Template + storageBackend *blob.Bucket + + redirectToBlobstore bool + redirectExpiry time.Duration } func (a *Application) Handler() http.Handler { r := mux.NewRouter() - r.NotFoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - rw.WriteHeader(http.StatusNotFound) - a.notFoundTmpl.Execute(rw, nil) - }) + renderTemplate := func(t *template.Template) http.HandlerFunc { + return func(rw http.ResponseWriter, r *http.Request) { + if err := t.Execute(rw, nil); err != nil { + log.Printf("rendering template: %v", err) + } + } + } + + r.NotFoundHandler = http.HandlerFunc(a.notFound) + r.HandleFunc("/", renderTemplate(a.indexTmpl)) + r.HandleFunc("/raw/{filename}", a.rawDownload) return r } +func (a *Application) notFound(rw http.ResponseWriter, r *http.Request) { + rw.WriteHeader(http.StatusNotFound) + if err := a.notFoundTmpl.Execute(rw, nil); err != nil { + log.Printf("rendering 404 template: %v", err) + } +} + +func (a *Application) internalError(rw http.ResponseWriter, r *http.Request) { + rw.Header().Set("Content-type", "text/plain") + rw.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(rw, "hammed server :(\n") +} + +func (a *Application) rawDownload(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + filename := vars["filename"] + + attrs, err := a.storageBackend.Attributes(ctx, filename) + switch gcerrors.Code(err) { + case gcerrors.NotFound: + a.notFound(rw, r) + return + default: + log.Printf("rawDownload(%q) Attributes: %v", filename, err) + a.internalError(rw, r) + return + case gcerrors.OK: + // OK + } + _ = attrs + + // Do things with attributes? + if a.redirectToBlobstore { + u, err := a.storageBackend.SignedURL(ctx, filename, &blob.SignedURLOptions{ + Expiry: a.redirectExpiry, + }) + switch gcerrors.Code(err) { + case gcerrors.NotFound: + // This is unlikely to get returned. + a.notFound(rw, r) + return + case gcerrors.OK: + http.Redirect(rw, r, u, http.StatusFound) + return + case gcerrors.Unimplemented: + // Fall back to serving directly. + default: + log.Printf("rawDownload(%q) SignedURL (continuing with fallback): %v", filename, err) + } + } + + // Perform direct serving. + a.rawDownloadDirect(ctx, rw, r, filename, attrs) +} + +func (a *Application) rawDownloadDirect(ctx context.Context, rw http.ResponseWriter, r *http.Request, filename string, attrs *blob.Attributes) { + // TODO(lukegb): Range header and conditionals?. + rdr, err := a.storageBackend.NewReader(ctx, filename, nil) + switch gcerrors.Code(err) { + case gcerrors.NotFound: + a.notFound(rw, r) + return + default: + a.internalError(rw, r) + return + case gcerrors.OK: + // OK + } + defer rdr.Close() + + if v := attrs.CacheControl; v != "" { + rw.Header().Set("Cache-Control", v) + } + if v := attrs.ContentDisposition; v != "" { + rw.Header().Set("Content-Disposition", v) + } + if v := attrs.ContentEncoding; v != "" { + rw.Header().Set("Content-Encoding", v) + } + if v := attrs.ContentLanguage; v != "" { + rw.Header().Set("Content-Language", v) + } + if v := attrs.ContentType; v != "" { + rw.Header().Set("Content-Type", v) + } + if v := attrs.ETag; v != "" { + rw.Header().Set("ETag", v) + } + if v := attrs.ModTime; !v.IsZero() { + rw.Header().Set("Last-Modified", v.UTC().Format(http.TimeFormat)) + } + + if v := attrs.Size; v != 0 { + rw.Header().Set("Content-Length", fmt.Sprintf("%d", v)) + } + + if _, err := io.Copy(rw, rdr); err != nil { + log.Printf("rawDownloadDirect(%q) copy: %v", filename, err) + } +} + func parseTemplate(t *template.Template, fsys fs.FS, name string) (*template.Template, error) { bs, err := fs.ReadFile(fsys, name) if err != nil { @@ -64,7 +194,22 @@ func loadTemplate(fsys fs.FS, name string, funcs template.FuncMap) (*template.Te } func New(ctx context.Context, cfg *Config) (*Application, error) { - a := new(Application) + a := &Application{ + redirectToBlobstore: cfg.RedirectToBlobstore, + redirectExpiry: cfg.RedirectExpiry, + } + if a.redirectExpiry == 0 { + a.redirectExpiry = defaultRedirectExpiry + } + + bkt := cfg.StorageBackend + if bkt == nil { + var err error + if bkt, err = blob.OpenBucket(ctx, cfg.StorageURL); err != nil { + return nil, fmt.Errorf("opening bucket %q: %v", cfg.StorageURL, err) + } + } + a.storageBackend = bkt tmpls := []struct { t **template.Template