go: add vault/vaultgcsblob to create vaultgs:// scheme which uses vault

go/default.nix

@@ -13,4 +13,5 @@ args: {
   secretsmgr = import ./secretsmgr args;
   tokend = import ./tokend args;
   access = import ./access args;
+  vault = import ./vault args;
 }

go/nix/bcachegc/bcachegc.go

@@ -21,6 +21,7 @@ import (
 	"hg.lukegb.com/lukegb/depot/go/nix/nar/narinfo"
 
 	_ "gocloud.dev/blob/gcsblob"
+	_ "hg.lukegb.com/lukegb/depot/go/vault/vaultgcsblob"
 )
 
 var (

go/nix/bcachegc/default.nix

@@ -14,5 +14,6 @@ depot.third_party.buildGo.program {
     third_party.gopkgs."golang.org".x.sync.errgroup
     third_party.gopkgs."golang.org".x.sync.singleflight
     go.nix.nar.narinfo
+    go.vault.vaultgcsblob
   ];
 }

go/nix/bcacheup/bcacheup.go

@@ -27,6 +27,7 @@ import (
 
 	_ "gocloud.dev/blob/fileblob"
 	_ "gocloud.dev/blob/gcsblob"
+	_ "hg.lukegb.com/lukegb/depot/go/vault/vaultgcsblob"
 )
 
 var (

go/nix/bcacheup/default.nix

@@ -18,5 +18,6 @@ depot.third_party.buildGo.program {
     go.nix.nar
     go.nix.nar.narinfo
     go.nix.nixstore
+    go.vault.vaultgcsblob
   ];
 }

go/vault/default.nix

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: 2023 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+args:
+{
+  vaultgcp = import ./vaultgcp args;
+  vaultgcsblob = import ./vaultgcsblob args;
+}

go/vault/vaultgcp/default.nix

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: 2023 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.package {
+  name = "vaultgcp";
+  path = "hg.lukegb.com/lukegb/depot/go/vault/vaultgcp";
+  srcs = [
+    ./token.go
+  ];
+  deps = with depot; [
+    third_party.gopkgs."golang.org".x.oauth2
+  ];
+}

go/vault/vaultgcp/token.go

@@ -0,0 +1,109 @@
+// Package vaultgcp allows fetching GCP service account credentials from Vault.
+package vaultgcp
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"golang.org/x/oauth2"
+)
+
+type tokenSource struct {
+	ctx context.Context
+
+	VaultAddress string
+	VaultPath    string
+}
+
+func (s tokenSource) Token() (*oauth2.Token, error) {
+	return Token(s.ctx, s.VaultAddress, s.VaultPath)
+}
+
+// TokenSource returns an oauth2.TokenSource that fetchs OAuth2 tokens from the specified Vault address and Vault path.
+func TokenSource(ctx context.Context, vaultAddr string, vaultPath string) oauth2.TokenSource {
+	return oauth2.ReuseTokenSource(nil, &tokenSource{
+		ctx:          ctx,
+		VaultAddress: vaultAddr,
+		VaultPath:    vaultPath,
+	})
+}
+
+// Token returns an OAuth token held by a GCP roleset in Vault.
+// vaultPath should look like e.g. gcp/roleset/binary-cache-deployer/token.
+func Token(ctx context.Context, vaultAddr string, vaultPath string) (*oauth2.Token, error) {
+	resp, err := vaultGet(ctx, vaultAddr, vaultPath)
+	if err != nil {
+		return nil, fmt.Errorf("fetching %v from %v: %w", vaultAddr, vaultPath, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("bad status code %v %s from Vault", resp.StatusCode, resp.Status)
+	}
+
+	type vaultResponse struct {
+		Data struct {
+			ExpiresAtSeconds int64  `json:"expires_at_seconds"`
+			Token            string `json:"token"`
+		} `json:"data"`
+		Errors []string `json:"errors"`
+	}
+	var vr vaultResponse
+	if err := json.NewDecoder(resp.Body).Decode(&vr); err != nil {
+		return nil, fmt.Errorf("parsing Vault response: %w", err)
+	}
+
+	if len(vr.Errors) > 0 {
+		return nil, fmt.Errorf("Vault returned errors: %q", vr.Errors)
+	}
+	if vr.Data.Token == "" {
+		return nil, fmt.Errorf("Vault returned no errors, but also no tokens")
+	}
+	var expiryTime time.Time
+	if vr.Data.ExpiresAtSeconds != 0 {
+		expiryTime = time.Unix(vr.Data.ExpiresAtSeconds, 0)
+	}
+	return &oauth2.Token{
+		AccessToken: vr.Data.Token,
+		Expiry:      expiryTime,
+	}, nil
+}
+
+func vaultGet(ctx context.Context, vaultAddrStr string, vaultPath string) (*http.Response, error) {
+	vaultAddr, err := url.Parse(vaultAddrStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing Vault address %q: %w", vaultAddrStr, err)
+	}
+
+	vaultClient := http.DefaultClient
+	vaultURL := vaultAddrStr
+	if vaultAddr.Scheme == "unix" {
+		vaultURL = "http://vault"
+		vaultClient = &http.Client{
+			Transport: &http.Transport{
+				DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+					d := &net.Dialer{}
+					return d.DialContext(ctx, "unix", vaultAddr.Path)
+				},
+				MaxIdleConns:          1,
+				IdleConnTimeout:       90 * time.Second,
+				TLSHandshakeTimeout:   10 * time.Second,
+				ExpectContinueTimeout: 1 * time.Second,
+			},
+		}
+	} else {
+		log.Printf("falling back to http.DefaultClient for talking to Vault; this is unlikely to work")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/v1/%s", vaultURL, vaultPath), nil)
+	if err != nil {
+		return nil, fmt.Errorf("formulating Vault request: %w", err)
+	}
+	req.Header.Set("X-Vault-Request", "true")
+	return vaultClient.Do(req)
+}

go/vault/vaultgcsblob/default.nix

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2023 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.package {
+  name = "vaultgcsblob";
+  path = "hg.lukegb.com/lukegb/depot/go/vault/vaultgcsblob";
+  srcs = [
+    ./vaultgcsblob.go
+  ];
+  deps = with depot; [
+    go.vault.vaultgcp
+    third_party.gopkgs."gocloud.dev".blob
+    third_party.gopkgs."gocloud.dev".blob.gcsblob
+    third_party.gopkgs."gocloud.dev".gcp
+  ];
+}

go/vault/vaultgcsblob/vaultgcsblob.go

@@ -0,0 +1,47 @@
+// Package vaultgcsblob registers the vaultgs:// URL scheme with gocloud.dev.
+package vaultgcsblob
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/url"
+	"sync"
+
+	"gocloud.dev/blob"
+	"gocloud.dev/blob/gcsblob"
+	"gocloud.dev/gcp"
+	"hg.lukegb.com/lukegb/depot/go/vault/vaultgcp"
+)
+
+var (
+	vaultAddr        = flag.String("vault_addr", "unix:///run/tokend/sock", "Address of vault agent, or tokend.")
+	vaultTokenSource = flag.String("vault_token_source", "", "Token source for retrieving OAuth token. e.g. gcp/roleset/binary-cache-deployer/token")
+)
+
+type lazyOpener struct {
+	init   sync.Once
+	opener *gcsblob.URLOpener
+	err    error
+}
+
+func (o *lazyOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) {
+	o.init.Do(func() {
+		c, err := gcp.NewHTTPClient(gcp.DefaultTransport(), vaultgcp.TokenSource(ctx, *vaultAddr, *vaultTokenSource))
+		if err != nil {
+			o.err = fmt.Errorf("creating GCP HTTP client using Vault token: %w", err)
+			return
+		}
+		o.opener = &gcsblob.URLOpener{
+			Client: c,
+		}
+	})
+	if o.err != nil {
+		return nil, o.err
+	}
+	return o.opener.OpenBucketURL(ctx, u)
+}
+
+func init() {
+	blob.DefaultURLMux().RegisterBucket("vaultgs", new(lazyOpener))
+}