From b7cd0d0e297f1ca8b809d64fed27bdbc8c30bac5 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 21 Mar 2021 18:52:22 +0000 Subject: [PATCH] fup: add cheddar as a syntax highlighter --- web/fup/cmd/serve.go | 35 +++++- web/fup/fuphttp/fuphttp.go | 11 ++ web/fup/minicheddar/minicheddar.go | 171 +++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 web/fup/minicheddar/minicheddar.go diff --git a/web/fup/cmd/serve.go b/web/fup/cmd/serve.go index ef4b7f5a79..48ba35475a 100644 --- a/web/fup/cmd/serve.go +++ b/web/fup/cmd/serve.go @@ -9,12 +9,14 @@ import ( "fmt" "log" "net/http" + "os/exec" "strings" "github.com/google/safehtml" "github.com/spf13/cobra" "hg.lukegb.com/lukegb/depot/web/fup/fuphttp" "hg.lukegb.com/lukegb/depot/web/fup/fupstatic" + "hg.lukegb.com/lukegb/depot/web/fup/minicheddar" ) func init() { @@ -24,13 +26,18 @@ func init() { serveCmd.Flags().StringVar(&serveStaticRoot, "static-root", "/static/", "Root address from which static assets should be referenced.") serveCmd.Flags().StringVarP(&serveBind, "listen", "l", ":8191", "Bind address for HTTP server.") serveCmd.Flags().BoolVar(&serveDirectOnly, "direct-only", false, "If set, all file serving will be proxied, even if the backend supports signed URLs.") + + serveCmd.Flags().StringVar(&serveCheddarPath, "cheddar-path", "cheddar", "Path to 'cheddar' binary to use for syntax highlighting. If it cannot be found, syntax highlighting and markdown rendering will be disabled.") + serveCmd.Flags().StringVar(&serveCheddarAddr, "cheddar-address", "", "If non-empty, will be used instead of attempting to spawn a copy of cheddar.") } var ( - serveBind string - serveRoot string - serveStaticRoot string - serveDirectOnly bool + serveBind string + serveRoot string + serveStaticRoot string + serveDirectOnly bool + serveCheddarPath string + serveCheddarAddr string serveCmd = &cobra.Command{ Use: "serve", @@ -44,6 +51,11 @@ var ( } ctx := context.Background() + highlighter, err := serveCheddar(ctx) + if err != nil { + return fmt.Errorf("spawning cheddar syntax highlighter: %v", err) + } + cfg := &fuphttp.Config{ Templates: fupstatic.Templates, Static: fupstatic.Static, @@ -51,6 +63,7 @@ var ( AppRoot: serveRoot, StorageURL: bucketURL, RedirectToBlobstore: !serveDirectOnly, + Highlighter: highlighter, } a, err := fuphttp.New(ctx, cfg) if err != nil { @@ -63,3 +76,17 @@ var ( }, } ) + +func serveCheddar(ctx context.Context) (*minicheddar.Cheddar, error) { + if serveCheddarAddr != "" { + return minicheddar.Remote(serveCheddarAddr), nil + } + + cpath, err := exec.LookPath(serveCheddarPath) + if err != nil { + log.Printf("couldn't find cheddar at %q; disabling syntax highlighting", serveCheddarPath) + return nil, nil + } + + return minicheddar.Spawn(ctx, cpath) +} diff --git a/web/fup/fuphttp/fuphttp.go b/web/fup/fuphttp/fuphttp.go index 620ffa1078..311691c41b 100644 --- a/web/fup/fuphttp/fuphttp.go +++ b/web/fup/fuphttp/fuphttp.go @@ -27,6 +27,11 @@ const ( defaultRedirectExpiry = 5 * time.Minute ) +type Highlighter interface { + Markdown(ctx context.Context, text string) (safehtml.HTML, error) + Code(ctx context.Context, filename, theme, text string) (safehtml.HTML, error) +} + type Config struct { Templates fs.FS Static fs.FS @@ -48,6 +53,10 @@ type Config struct { // FilenameGenerator returns a new filename based on the provided prefix and extension. FilenameGenerator fngen.FilenameGenerator + // Highlighter is used for syntax highlighting and Markdown rendering. + // If nil, then no syntax highlighting or Markdown rendering will be performed. + Highlighter Highlighter + // UseDirectDownload decides whether the "pretty" wrapped page or the direct download page is the most appropriate for a given set of parameters. UseDirectDownload func(fileExtension string, mimeType string) bool } @@ -59,6 +68,8 @@ type Application struct { appRoot string + highlighter Highlighter + redirectToBlobstore bool redirectExpiry time.Duration diff --git a/web/fup/minicheddar/minicheddar.go b/web/fup/minicheddar/minicheddar.go new file mode 100644 index 0000000000..f4345cda1b --- /dev/null +++ b/web/fup/minicheddar/minicheddar.go @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2021 Luke Granger-Brown +// +// SPDX-License-Identifier: Apache-2.0 + +package minicheddar + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os/exec" + "time" + + "github.com/google/safehtml" + "github.com/google/safehtml/uncheckedconversions" +) + +const startupMax = 10 * time.Second + +type Cheddar struct { + remote string + stop func() +} + +type markdownReqResp struct { + Data string `json:"data"` +} + +func (c *Cheddar) Markdown(ctx context.Context, text string) (safehtml.HTML, error) { + buf := new(bytes.Buffer) + + if err := json.NewEncoder(buf).Encode(markdownReqResp{Data: text}); err != nil { + return safehtml.HTML{}, fmt.Errorf("encoding markdown rendering request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/markdown", c.remote), buf) + if err != nil { + return safehtml.HTML{}, fmt.Errorf("making markdown request object: %w", err) + } + req.Header.Set("Content-type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return safehtml.HTML{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return safehtml.HTML{}, fmt.Errorf("markdown rendering request returned status %s; not OK", resp.StatusCode) + } + + var respData markdownReqResp + if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { + return safehtml.HTML{}, fmt.Errorf("decoding markdown rendering response: %w", err) + } + + return uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(respData.Data), nil +} + +type codeReq struct { + Filepath string `json:"filepath"` + Theme string `json:"theme"` + Code string `json:"code"` +} + +type codeResp struct { + IsPlaintext bool `json:"is_plaintext"` + Data string `json:"data"` +} + +func (c *Cheddar) Code(ctx context.Context, filename, theme, text string) (safehtml.HTML, error) { + buf := new(bytes.Buffer) + + if err := json.NewEncoder(buf).Encode(codeReq{ + Filepath: filename, + Theme: theme, + Code: text, + }); err != nil { + return safehtml.HTML{}, fmt.Errorf("encoding code rendering request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s", c.remote), buf) + if err != nil { + return safehtml.HTML{}, fmt.Errorf("making code request object: %w", err) + } + req.Header.Set("Content-type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return safehtml.HTML{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return safehtml.HTML{}, fmt.Errorf("code rendering request returned status %s; not OK", resp.StatusCode) + } + + var respData codeResp + if err := json.NewDecoder(resp.Body).Decode(&respData); err != nil { + return safehtml.HTML{}, fmt.Errorf("decoding code rendering response: %w", err) + } + + if respData.IsPlaintext { + return safehtml.HTMLEscaped(respData.Data), nil + } + return uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(respData.Data), nil +} + +func (c *Cheddar) healthCheck(ctx context.Context) error { + _, err := c.Markdown(ctx, "") + return err +} + +func (c *Cheddar) Close() { + if c.stop != nil { + c.stop() + } +} + +// Remote connects to an already running instance of Cheddar. +func Remote(url string) *Cheddar { + return &Cheddar{ + remote: url, + } +} + +// Spawn spawns a new instance of Cheddar listening on some port. +// This is potentially race-y, because it relies on binding, then unbinding and spawning Cheddar. +// But... it should be good enough. +func Spawn(ctx context.Context, cpath string) (*Cheddar, error) { + l, err := net.Listen("tcp", "[::1]:0") + if err != nil { + return nil, fmt.Errorf("finding a port for cheddar: %w", err) + } + addr := l.Addr().(*net.TCPAddr) + port := addr.Port + if err := l.Close(); err != nil { + return nil, fmt.Errorf("closing port-sentinel listener: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, cpath, "--sourcegraph-server", "--listen", fmt.Sprintf("[::1]:%d", port)) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("launching cheddar: %w", err) + } + c := &Cheddar{ + remote: fmt.Sprintf("http://[::1]:%d", port), + stop: cancel, + } + + startupCtx, startupCancel := context.WithTimeout(ctx, startupMax) + defer startupCancel() + + // Give cheddar a bit to start up before we start healthchecking it. + time.Sleep(1 * time.Millisecond) + for { + if err := c.healthCheck(startupCtx); errors.Is(err, context.DeadlineExceeded) { + cancel() // Kill the subprocess. + return nil, err + } else if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + + return c, nil +}