go/minotarproxy: import

go/default.nix

@@ -5,4 +5,5 @@
 args: {
   twitterchiver = import ./twitterchiver args;
   openshiftauth = import ./openshiftauth args;
+  minotarproxy = import ./minotarproxy args;
 }

go/minotarproxy/default.nix

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.program {
+  name = "minotarproxy";
+  srcs = [ ./minotarproxy.go ];
+  deps = with depot.third_party; [
+    gopkgs."github.com".golang.glog
+    gopkgs."github.com".google.uuid
+    gopkgs."github.com".prometheus.client_golang.prometheus
+    gopkgs."github.com".prometheus.client_golang.prometheus.promhttp
+    gopkgs."golang.org".x.crypto.acme.autocert
+  ];
+}

go/minotarproxy/minotarproxy.go

@@ -0,0 +1,287 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/crypto/acme/autocert"
+)
+
+var (
+	serverBind   = flag.String("server_bind", "185.230.222.225:13231", "IP address/port to bind proxy server to")
+	outboundBind = flag.String("outbound_bind", generateOutbound(), "IP addresses to bind outbound requests to")
+	outboundMap  = flag.String("outbound_map", "/users/=https://api.mojang.com/users/,/session/=https://sessionserver.mojang.com/session/", "Comma-separated mapping of [path prefix]=[remote base URL]")
+
+	autocertDomain       = flag.String("autocert_domain", "", "If non-empty, enables autocert for the provided domain.")
+	autocertInsecureBind = flag.String("autocert_insecure_bind", "185.230.222.225:80", "Port 80 IP to bind for HTTP-01 ACME challenge.")
+	autocertCacheDir     = flag.String("autocert_cache_dir", "/home/minotarproxy/autocertcache", "Autocert cache directory.")
+)
+
+func generateOutbound() string {
+	var s []string
+	for n := 225; n <= 253; n++ {
+		s = append(s, fmt.Sprintf("185.230.222.%d", n))
+	}
+	return strings.Join(s, ",")
+}
+
+type roundRobinMetrics struct {
+	mInflight  *prometheus.GaugeVec
+	mLatency   *prometheus.HistogramVec
+	mRequests  *prometheus.CounterVec
+	mResponses *prometheus.CounterVec
+}
+
+type roundRobinRoundtripper struct {
+	mu   sync.Mutex
+	sip  []string
+	pool []http.RoundTripper
+	last int
+
+	m *roundRobinMetrics
+}
+
+func (rt *roundRobinRoundtripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	rt.mu.Lock()
+	rt.last = (rt.last + 1) % len(rt.pool)
+	rt.mu.Unlock()
+
+	handler := handlerName(req.Context())
+	labels := prometheus.Labels{"handler": handler, "sourceIP": rt.sip[rt.last]}
+	gi := rt.m.mInflight.With(labels)
+	gi.Inc()
+	defer gi.Dec()
+	hl := rt.m.mLatency.With(labels)
+	rt.m.mRequests.With(labels).Inc()
+	cvr, err := rt.m.mResponses.CurryWith(labels)
+	if err != nil {
+		return nil, err
+	}
+
+	startTime := time.Now()
+	resp, err := rt.pool[rt.last].RoundTrip(req)
+	latency := time.Now().Sub(startTime)
+
+	hl.Observe(latency.Seconds())
+	cvr.With(prometheus.Labels{"responseCode": fmt.Sprintf("%d", resp.StatusCode)}).Inc()
+
+	return resp, err
+}
+
+func NewRoundRobinRoundtripper(ips []string, m *roundRobinMetrics) (*roundRobinRoundtripper, error) {
+	var pool []http.RoundTripper
+	var sip []string
+	for _, oip := range ips {
+		a, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%v:0", oip))
+		if err != nil {
+			return nil, err
+		}
+		pool = append(pool, &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 120 * time.Second,
+				DualStack: true,
+				LocalAddr: a,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		})
+		sip = append(sip, oip)
+	}
+
+	return &roundRobinRoundtripper{
+		pool: pool,
+		sip:  sip,
+		m:    m,
+	}, nil
+}
+
+type key int
+
+const requestIDKey key = 0
+const requestStartKey key = 1
+const handlerKey key = 2
+
+func newRequestID() string {
+	return uuid.New().String()
+}
+
+func requestID(ctx context.Context) string {
+	return ctx.Value(requestIDKey).(string)
+}
+
+func timeSinceStart(ctx context.Context) time.Duration {
+	return time.Now().Sub(ctx.Value(requestStartKey).(time.Time))
+}
+
+func handlerName(ctx context.Context) string {
+	return ctx.Value(handlerKey).(string)
+}
+
+func requestIDMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		ctx = context.WithValue(ctx, requestIDKey, newRequestID())
+		ctx = context.WithValue(ctx, requestStartKey, time.Now())
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func attachHandlerMiddleware(next http.Handler, handler string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		ctx = context.WithValue(ctx, handlerKey, handler)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func newReverseProxy(baseURL *url.URL, rrrt http.RoundTripper) http.Handler {
+	rp := httputil.NewSingleHostReverseProxy(baseURL)
+	d := rp.Director
+	rp.Director = func(req *http.Request) {
+		d(req)
+		req.Header.Set("Host", req.URL.Host)
+		req.Host = req.URL.Host
+		glog.Infof("REQ [%v] [%v] %v %v (%v)", requestID(req.Context()), req.RemoteAddr, req.Method, req.URL, req.Header.Get("User-Agent"))
+	}
+	rp.ModifyResponse = func(resp *http.Response) error {
+		req := resp.Request
+		ctx := req.Context()
+		glog.Infof("RESP [%v] [%v] %v %v (%v): [%v] [%v]", requestID(ctx), req.RemoteAddr, req.Method, req.URL, req.Header.Get("User-Agent"), timeSinceStart(ctx), resp.Status)
+		return nil
+	}
+	rp.Transport = rrrt
+	return http.StripPrefix(baseURL.Path, rp)
+}
+
+type listener struct {
+	m    *autocert.Manager
+	conf *tls.Config
+
+	tcpListener  net.Listener
+	tcpListenErr error
+}
+
+func (ln *listener) Accept() (net.Conn, error) {
+	if ln.tcpListenErr != nil {
+		return nil, ln.tcpListenErr
+	}
+	conn, err := ln.tcpListener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	tcpConn := conn.(*net.TCPConn)
+	tcpConn.SetKeepAlive(true)
+	tcpConn.SetKeepAlivePeriod(3 * time.Minute)
+	return tls.Server(tcpConn, ln.conf), nil
+}
+
+func (ln *listener) Addr() net.Addr {
+	if ln.tcpListener != nil {
+		return ln.tcpListener.Addr()
+	}
+	// net.Listen failed. Return something non-nil in case callers
+	// call Addr before Accept:
+	return &net.TCPAddr{IP: net.IP{0, 0, 0, 0}, Port: 443}
+}
+
+func (ln *listener) Close() error {
+	if ln.tcpListenErr != nil {
+		return ln.tcpListenErr
+	}
+	return ln.tcpListener.Close()
+}
+
+func main() {
+	flag.Parse()
+
+	m := &roundRobinMetrics{
+		mInflight: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "proxy_inflight",
+				Help: "Count of current inflight HTTP requests, per source-IP",
+			},
+			[]string{"handler", "sourceIP"},
+		),
+		mLatency: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: "proxy_latency",
+				Help: "Histogram of request latency distributions",
+			},
+			[]string{"handler", "sourceIP"},
+		),
+		mRequests: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "proxy_requests",
+				Help: "Count of requests that have passed through this proxy",
+			},
+			[]string{"handler", "sourceIP"},
+		),
+		mResponses: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "proxy_responses",
+				Help: "Count of responses that have passed through this proxy",
+			},
+			[]string{"handler", "sourceIP", "responseCode"},
+		),
+	}
+	prometheus.MustRegister(m.mInflight, m.mLatency, m.mRequests, m.mResponses)
+
+	outboundIPs := strings.Split(*outboundBind, ",")
+	rrrt, err := NewRoundRobinRoundtripper(outboundIPs, m)
+	if err != nil {
+		glog.Exitf("NewRoundRobinRoundtripper: %v", err)
+	}
+
+	remoteMaps := strings.Split(*outboundMap, ",")
+	for _, ms := range remoteMaps {
+		bits := strings.Split(ms, "=")
+		remoteURL, err := url.Parse(bits[1])
+		if err != nil {
+			glog.Exitf("url.Parse(%q): %v", bits[1], err)
+		}
+		localPath := bits[0]
+
+		rp := newReverseProxy(remoteURL, rrrt)
+		http.Handle(localPath, attachHandlerMiddleware(requestIDMiddleware(rp), bits[1]))
+	}
+
+	http.Handle("/metrics", promhttp.Handler())
+	glog.Info("Initialization complete. Ready to go.")
+	if *autocertDomain != "" {
+		m := &autocert.Manager{
+			Prompt:     autocert.AcceptTOS,
+			HostPolicy: autocert.HostWhitelist(*autocertDomain),
+			Cache:      autocert.DirCache(*autocertCacheDir),
+		}
+		ln := &listener{
+			m: m,
+			conf: &tls.Config{
+				GetCertificate: m.GetCertificate,
+				NextProtos:     []string{"h2", "http/1.1"},
+			},
+		}
+		ln.tcpListener, ln.tcpListenErr = net.Listen("tcp", *serverBind)
+		go func() {
+			glog.Fatal(http.ListenAndServe(*autocertInsecureBind, m.HTTPHandler(nil)))
+		}()
+		glog.Fatal(http.Serve(ln, nil))
+	} else {
+		glog.Fatal(http.ListenAndServe(*serverBind, nil))
+	}
+}

third_party/gopkgs/github.com/beorn7/perks/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/beorn7/perks";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "beorn7";
+    repo = "perks";
+    rev = "37c8de3658fcb183f997c4e13e8337516ab753e6";
+    hash = "sha256:17n4yygjxa6p499dj3yaqzfww2g7528165cl13haj97hlx94dgl7";
+  };
+}

third_party/gopkgs/github.com/cespare/xxhash/v2/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/cespare/xxhash/v2";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "cespare";
+    repo = "xxhash";
+    rev = "3b82fb7d186719faeedd0c2864f868c74fbf79a1";
+    hash = "sha256:1cpxfjamywjvm1530cdg7li60yabgby1a4ispina1a39h1f3c92k";
+  };
+}

third_party/gopkgs/github.com/golang/glog/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/golang/glog";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "golang";
+    repo = "glog";
+    rev = "23def4e6c14b4da8ac2ed8007337bc5eb5007998";
+    hash = "sha256:0jb2834rw5sykfr937fxi8hxi2zy80sj2bdn9b3jb4b26ksqng30";
+  };
+}

third_party/gopkgs/github.com/golang/protobuf/default.nix

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/golang/protobuf";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "golang";
+    repo = "protobuf";
+    rev = "4846b58453b3708320bdb524f25cc5a1d9cda4d4";
+    hash = "sha256:1kf1d7xmyjvy0z6j5czp6nqyvj9zrk6liv6znif08927xqfrzyln";
+  };
+  deps = with depot.third_party.gopkgs."google.golang.org".protobuf; [
+    encoding.prototext
+    encoding.protowire
+    proto
+    reflect.protoreflect
+    reflect.protoregistry
+    runtime.protoiface
+    runtime.protoimpl
+    types.known.timestamppb
+    types.known.anypb
+    types.known.durationpb
+  ];
+}

third_party/gopkgs/github.com/google/uuid/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/google/uuid";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "google";
+    repo = "uuid";
+    rev = "0e4e31197428a347842d152773b4cace4645ca25";
+    hash = "sha256:1rbpfa0v0ly9sdnixcxhf79swki54ikgm1zkwwkj64p1ws66syqd";
+  };
+}

third_party/gopkgs/github.com/matttproud/golang_protobuf_extensions/default.nix

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/matttproud/golang_protobuf_extensions";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "matttproud";
+    repo = "golang_protobuf_extensions";
+    rev = "c182affec369e30f25d3eb8cd8a478dee585ae7d";
+    hash = "sha256:1xqsf9vpcrd4hp95rl6kgmjvkv1df4aicfw4l5vfcxcwxknfx2xs";
+  };
+  deps = with depot.third_party; [
+    gopkgs."github.com".golang.protobuf.proto
+  ];
+}

third_party/gopkgs/github.com/prometheus/client_golang/default.nix

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/prometheus/client_golang";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "prometheus";
+    repo = "client_golang";
+    rev = "06b1a0a6ae29dd8b39953dc7f1954a0b2fd680be";
+    hash = "sha256:1a7ic0llbqypp8rgw4ax472nwiqm28y736qfbiscxc6lzkgzhdir";
+  };
+  deps = with depot.third_party; [
+    gopkgs."github.com".beorn7.perks.quantile
+    gopkgs."github.com".cespare.xxhash.v2
+    gopkgs."github.com".golang.protobuf.proto
+    gopkgs."github.com".golang.protobuf.ptypes
+    gopkgs."github.com".prometheus.client_model.go
+    gopkgs."github.com".prometheus.common.expfmt
+    gopkgs."github.com".prometheus.common.model
+    gopkgs."github.com".prometheus.procfs
+  ];
+}

third_party/gopkgs/github.com/prometheus/client_model/default.nix

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/prometheus/client_model";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "prometheus";
+    repo = "client_model";
+    rev = "60555c9708c786597e6b07bf846d0dc5c2a46f54";
+    hash = "sha256:1cdwzjm92zv2lacnmyw7rnmvay0hz4rknc5vs15r9jp7k0rai5yy";
+  };
+  deps = with depot.third_party; [
+    gopkgs."github.com".golang.protobuf.proto
+    gopkgs."github.com".golang.protobuf.ptypes.timestamp
+  ];
+}

third_party/gopkgs/github.com/prometheus/common/default.nix

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/prometheus/common";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "prometheus";
+    repo = "common";
+    rev = "317b7b125e8fddda956d0c9574e5f03f438ed5bc";
+    hash = "sha256:0idhigx8qfszf5n71pvsdycfh5xqwzhdsbhxfyyknbyv2954vya6";
+  };
+  deps = with depot.third_party; [
+    gopkgs."github.com".golang.protobuf.proto
+    gopkgs."github.com".golang.protobuf.ptypes
+    gopkgs."github.com".matttproud.golang_protobuf_extensions.pbutil
+    gopkgs."github.com".prometheus.client_model.go
+  ];
+}

third_party/gopkgs/github.com/prometheus/procfs/default.nix

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "github.com/prometheus/procfs";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "prometheus";
+    repo = "procfs";
+    rev = "910e68572b35da779e4f84cfa5261b0a67532d05";
+    hash = "sha256:0qdv4ybjis4hpr8infgv3l4haynnzjx07z8qhcrd1n556ayhl082";
+  };
+  deps = with depot.third_party; [
+    gopkgs."golang.org".x.sys.unix
+  ];
+}

third_party/gopkgs/golang.org/x/crypto/default.nix

@@ -11,4 +11,7 @@ depot.third_party.buildGo.external {
     rev = "7f63de1d35b0f77fa2b9faea3e7deb402a2383c8";
     sha256 = "1dr89jfs4dmpr3jqfshqqvfpzzdx4r76nkzhrvmixfrmn6wxrnd1";
   };
+  deps = with depot.third_party; [
+    gopkgs."golang.org".x.net.idna
+  ];
 }

third_party/gopkgs/golang.org/x/net/default.nix

@@ -11,4 +11,9 @@ depot.third_party.buildGo.external {
     rev = "0a1ea396d57c75b04aff7cb73d169717064c2b8a";
     hash = "sha256:0wg1minaybg5nbqm1zjgjjr1ns8japq013j8srcybacmd90vrj2l";
   };
+  deps = with depot.third_party; [
+    gopkgs."golang.org".x.text.secure.bidirule
+    gopkgs."golang.org".x.text.unicode.bidi
+    gopkgs."golang.org".x.text.unicode.norm
+  ];
 }

third_party/gopkgs/golang.org/x/sync/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "golang.org/x/sync";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "golang";
+    repo = "sync";
+    rev = "67f06af15bc961c363a7260195bcd53487529a21";
+    hash = "sha256:093p4panc808ak5bamzz7m9nb0xxib7778jpnr6f0xkz1n4fzyw5";
+  };
+}

third_party/gopkgs/golang.org/x/sys/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "golang.org/x/sys";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "golang";
+    repo = "sys";
+    rev = "da207088b7d1961c44ec61e9dc15a73745693ab6";
+    hash = "sha256:1i8q11i494cp3qia8w64k69wlcy4lgw9yya79yv85dz4brry29l8";
+  };
+}

third_party/gopkgs/google.golang.org/protobuf/default.nix

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.external {
+  path = "google.golang.org/protobuf";
+  src = depot.third_party.nixpkgs.fetchFromGitHub {
+    owner = "protocolbuffers";
+    repo = "protobuf-go";
+    rev = "d3470999428befce9bbefe77980ff65ac5a494c4";
+    hash = "sha256:0sgwfkcr6n7m1ivyq34rz4rd6gm5pzswa73nvzj59dkaknj68xfb";
+  };
+}