tokend: init

tokend is responsible for issuing service-scoped tokens based on the token held
and generated by the Vault Agent.

It can also generate "server-user" scoped tokens, which exist for convenience's
sake: they are not a strong attestation of the user on the machine, and have
limited privileges compared to a Vault token issued using e.g. `vault login
-method=oidc`.

go/default.nix

@@ -11,4 +11,5 @@ args: {
   nhsenglandtests = import ./nhsenglandtests args;
   journal2clickhouse = import ./journal2clickhouse args;
   secretsmgr = import ./secretsmgr args;
+  tokend = import ./tokend args;
 }

go/secretsmgr/secretsmgr.go

@@ -31,6 +31,9 @@ import (
 )
 
 var (
+	vaultAddress      = flag.String("vault_address", "https://vault.int.lukegb.com", "Address of Vault")
+	vaultAgentAddress = flag.String("vault_agent_address", "unix:///run/vault-agent/sock", "Address of Vault agent")
+
 	signSSHHostKeys           = flag.Bool("sign_ssh_host_keys", true, "Sign SSH host keys with CA")
 	sshHostKeyCAPath          = flag.String("ssh_host_key_ca_path", "ssh-host", "Path that the SSH CA is mounted at")
 	sshHostKeyRole            = flag.String("ssh_host_key_role", hostname(), "Role to use for signing SSH host keys")
@@ -609,7 +612,7 @@ func main() {
 
 	cfg := vapi.DefaultConfig()
 	cfg.Address = "https://vault.int.lukegb.com"
-	cfg.AgentAddress = "http://localhost:8200"
+	cfg.AgentAddress = "unix:///run/vault-agent/sock"
 	cfg.MaxRetries = 0
 	cfg.Timeout = 15 * time.Minute
 	c, err := vapi.NewClient(cfg)

go/tokend/default.nix

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ depot, ... }:
+depot.third_party.buildGo.program {
+  name = "tokend";
+  srcs = [
+    ./tokend.go
+    ./tokencache.go
+    ./vaultissuer.go
+  ];
+  deps = with depot.third_party; [
+    gopkgs."github.com".golang.glog
+    gopkgs."github.com".hashicorp.vault.api
+  ];
+}

go/tokend/tokencache.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"os/user"
+	"sync"
+	"time"
+
+	log "github.com/golang/glog"
+	vapi "github.com/hashicorp/vault/api"
+)
+
+type TokenSecret = vapi.Secret
+
+type ttledSecret struct {
+	*TokenSecret
+
+	expiration time.Time
+
+	isRenewable    bool
+	renewThreshold time.Time
+}
+
+func wrapToken(s *TokenSecret) ttledSecret {
+	now := time.Now()
+
+	ttl := s.Auth.LeaseDuration
+	renewable := s.Auth.Renewable
+
+	var ttlBuffer time.Duration
+	if ttl < 300 {
+		// Give ourselves an extra two minute buffer for renewal.
+		ttlBuffer = 2 * time.Minute
+	}
+
+	return ttledSecret{
+		TokenSecret: s,
+
+		expiration: now.Add(time.Duration(ttl) * time.Second),
+
+		isRenewable:    renewable,
+		renewThreshold: now.Add(time.Duration(ttl/2) * time.Second).Add(-ttlBuffer),
+	}
+}
+
+func (s ttledSecret) Expired() bool {
+	// We use !After rather than Before so that if it's _exactly_ now then we still return true.
+	return !s.expiration.After(time.Now())
+}
+
+func (s ttledSecret) ShouldRenew() bool {
+	if !s.isRenewable {
+		return false
+	}
+
+	return time.Now().After(s.renewThreshold)
+}
+
+type tokenInteractor interface {
+	Revoke(ctx context.Context, tokenSecret *TokenSecret) error
+	Issue(ctx context.Context, username string, isPlainUser bool) (*TokenSecret, error)
+	Renew(ctx context.Context, tokenSecret *TokenSecret) (*TokenSecret, error)
+}
+
+type tokenUserCache struct {
+	l sync.RWMutex
+	m map[string]ttledSecret
+	i tokenInteractor
+}
+
+func newCache(i tokenInteractor) *tokenUserCache {
+	return &tokenUserCache{
+		m: make(map[string]ttledSecret),
+		i: i,
+	}
+}
+
+func (c *tokenUserCache) expire() (revoke map[string]*TokenSecret) {
+	c.l.Lock()
+	defer c.l.Unlock()
+
+	var remove []string
+	revoke = make(map[string]*TokenSecret)
+	for username, s := range c.m {
+		// If the token has expired, remove it.
+		if s.Expired() {
+			remove = append(remove, username)
+			continue
+		}
+
+		// If the user no longer exists on the system, revoke it.
+		if _, err := user.Lookup(username); err != nil {
+			log.Infof("token for %v will be revoked because user lookup returned error: %v", username, err)
+			remove = append(remove, username)
+			revoke[username] = s.TokenSecret
+			continue
+		}
+
+		// Otherwise, leave it alone.
+	}
+
+	for _, username := range remove {
+		delete(c.m, username)
+	}
+
+	return revoke
+}
+
+func (c *tokenUserCache) renew() {
+	ctx := context.Background()
+
+	toRenew := make(map[string]*TokenSecret)
+	c.l.Lock()
+	for username, s := range c.m {
+		if s.ShouldRenew() {
+			toRenew[username] = s.TokenSecret
+		}
+	}
+	c.l.Unlock()
+
+	for username, s := range toRenew {
+		log.Infof("renewing token for %v", username)
+		newS, err := c.i.Renew(ctx, s)
+		if err != nil {
+			log.Errorf("renewing token for %v: %w (discarding cached token)", username, err)
+
+			// Discard the token in cache.
+			c.l.Lock()
+			delete(c.m, username)
+			c.l.Unlock()
+			continue
+		}
+
+		c.l.Lock()
+		if oldS, ok := c.m[username]; ok && oldS.TokenSecret == s {
+			c.m[username] = wrapToken(newS)
+		} else if ok {
+			log.Warningf("after renewing token for %v discovered that the token in cache had changed in the meantime; dropping refreshed token")
+		}
+		c.l.Unlock()
+	}
+}
+
+func (c *tokenUserCache) tick(ctx context.Context) error {
+	log.Info("token cache is ticking")
+	revoke := c.expire()
+	for username, tokenSecret := range revoke {
+		if err := c.i.Revoke(ctx, tokenSecret); err != nil {
+			log.Errorf("unable to revoke token for %v: %w", username, err)
+		}
+	}
+
+	c.renew()
+
+	return nil
+}
+
+func (c *tokenUserCache) Get(ctx context.Context, username string, isPlainUser bool) (*TokenSecret, error) {
+	c.l.RLock()
+	token, ok := c.m[username]
+	c.l.RUnlock()
+
+	// If we got a token, but it's expired, delete it and pretend it didn't exist.
+	if ok && token.Expired() {
+		c.l.Lock()
+		delete(c.m, username)
+		c.l.Unlock()
+
+		ok = false
+	}
+
+	if ok {
+		// We have a non-expired pre-existing token!
+		return token.TokenSecret, nil
+	}
+
+	// Issue a new token.
+	issuedToken, err := c.i.Issue(ctx, username, isPlainUser)
+	if err != nil {
+		return nil, err
+	}
+
+	// OK, now we check if someone has issued a token for us in the
+	// meantime...  We could (but don't) do any more complex coordination
+	// than this, so we might actually issue two tokens for the same user
+	// concurrently. Even if we do this though, one of the tokens will
+	// "win" and we'll revoke the other one.
+	c.l.Lock()
+	defer c.l.Unlock()
+
+	if bgToken, ok := c.m[username]; ok && !bgToken.Expired() {
+		// Just use this one, I guess.
+		go c.i.Revoke(context.Background(), issuedToken)
+		return bgToken.TokenSecret, nil
+	}
+
+	token = wrapToken(issuedToken)
+	c.m[username] = token
+
+	return token.TokenSecret, nil
+}
+
+func (c *tokenUserCache) Purge(ctx context.Context, username string) {
+	c.l.Lock()
+	token, ok := c.m[username]
+	delete(c.m, username)
+	c.l.Unlock()
+
+	if !ok {
+		return
+	}
+
+	// Revoke the token as well, for good measure, in case the parent hasn't already done that.
+	c.i.Revoke(ctx, token.TokenSecret)
+}

go/tokend/tokend.go

@@ -0,0 +1,289 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/user"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/sys/unix"
+
+	log "github.com/golang/glog"
+	vapi "github.com/hashicorp/vault/api"
+)
+
+var (
+	listenPath = flag.String("listen_path", "/run/tokend/sock", "Path to listen for connections to tokend.")
+	agentAddr  = flag.String("agent_address", "unix:///run/vault-agent/sock", "Address of vault agent.")
+
+	cacheTickInterval = flag.Duration("cache_tick_interval", 1*time.Minute, "Time between checking for expirations.")
+	userGroup         = flag.String("user_group", "users", "Name of a group that indicates that the requesting user is a 'user' and not a service.")
+)
+
+type userContextKeyType struct{}
+
+var userContextKey = userContextKeyType{}
+
+type userData struct {
+	Username    string
+	IsPlainUser bool
+}
+
+type vaultProxier struct {
+	v  *vapi.Client
+	c  *tokenUserCache
+	hc *http.Client
+}
+
+func shouldAttachToken(path string) bool {
+	if path == "/v1/auth/token/revoke-self" {
+		return false
+	}
+	return true
+}
+
+func shouldObfuscateTokenResponse(r *http.Request, resp *http.Response, attachedToken bool) bool {
+	path := r.URL.Path
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return false
+	}
+	return attachedToken && (path == "/v1/auth/token/lookup-self" || path == "/v1/auth/token/renew-self")
+}
+
+func obfuscateAndCopyResponse(w io.Writer, r io.Reader) error {
+	sec, err := vapi.ParseSecret(r)
+	if err != nil {
+		return err
+	}
+
+	delete(sec.Data, "id")
+	delete(sec.Data, "accessor")
+
+	if sec.Auth != nil {
+		sec.Auth.ClientToken = ""
+		sec.Auth.Accessor = ""
+	}
+
+	return json.NewEncoder(w).Encode(sec)
+}
+
+func (vp *vaultProxier) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	u, ok := ctx.Value(userContextKey).(*userData)
+	if !ok {
+		http.Error(rw, "no username could be determined from the request", http.StatusInternalServerError)
+		return
+	}
+
+	t, err := vp.c.Get(ctx, u.Username, u.IsPlainUser)
+	if err != nil {
+		http.Error(rw, fmt.Sprintf("fetching token for %s: %v", u.Username, err), http.StatusInternalServerError)
+		return
+	}
+
+	tokenStr, err := t.TokenID()
+	if err != nil {
+		http.Error(rw, fmt.Sprintf("extracting token string for %s: %v", u.Username, err), http.StatusInternalServerError)
+		return
+	}
+
+	outReq := r.Clone(ctx)
+	outReq.RequestURI = ""
+	outReq.URL.Scheme = "http"
+	outReq.URL.Host = "vault-agent"
+	outReq.Trailer = nil
+
+	attachedToken := false
+	if shouldAttachToken(r.URL.Path) {
+		outReq.Header.Set("X-Vault-Token", tokenStr)
+		attachedToken = true
+	}
+	log.Infof("incoming request [%v / isPlainUser=%v] %v %v", u.Username, u.IsPlainUser, r.Method, r.URL.Path)
+	start := time.Now()
+
+	resp, err := vp.hc.Do(outReq)
+	if err != nil {
+		http.Error(rw, fmt.Sprintf("making backend request to vault agent: %v", err), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	log.Infof("outgoing response [%v / isPlainUser=%v] %v %v: %v %v", u.Username, u.IsPlainUser, r.Method, r.URL.Path, resp.StatusCode, time.Now().Sub(start))
+
+	for k, vs := range resp.Header {
+		rw.Header()[k] = vs
+	}
+	if shouldObfuscateTokenResponse(r, resp, attachedToken) {
+		rw.Header().Del("Content-Length")
+		rw.WriteHeader(resp.StatusCode)
+		if err := obfuscateAndCopyResponse(rw, resp.Body); err != nil {
+			log.Errorf("copying obfuscated lookup-self response: %v", err)
+		}
+	} else {
+		rw.WriteHeader(resp.StatusCode)
+		if _, err := io.Copy(rw, resp.Body); err != nil {
+			log.Errorf("copying response from agent: %v", err)
+		}
+	}
+}
+
+var (
+	userGroupGidSaved string
+	userGroupGidOnce  sync.Once
+)
+
+func userGroupGid() string {
+	userGroupGidOnce.Do(func() {
+		userGroupGidSaved = ""
+		if *userGroup == "" {
+			// Disabled.
+			return
+		}
+		g, err := user.LookupGroup(*userGroup)
+		if err != nil {
+			log.Errorf("looking up user group %q: %v", *userGroup, err)
+			return
+		}
+		userGroupGidSaved = g.Gid
+	})
+	return userGroupGidSaved
+}
+
+func attachUserData(ctx context.Context, c net.Conn) context.Context {
+	uc, ok := c.(*net.UnixConn)
+	if !ok {
+		log.Warningf("asked to attachUserData to a non UnixConn (%T)", c)
+		return ctx
+	}
+
+	raw, err := uc.SyscallConn()
+	if err != nil {
+		log.Warningf("unable to get the underlying raw connection for UnixConn")
+		return ctx
+	}
+
+	var cred *unix.Ucred
+	if ctrlErr := raw.Control(func(fd uintptr) {
+		cred, err = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED)
+	}); ctrlErr != nil {
+		log.Warningf("control operation to get username failed: %v", err)
+		return ctx
+	} else if err != nil {
+		log.Warningf("getsockoptucred failed: %v", err)
+		return ctx
+	}
+
+	u, err := user.LookupId(strconv.FormatUint(uint64(cred.Uid), 10))
+	if err != nil {
+		log.Warningf("looking up UID %d from unix socket: %v", cred.Uid, err)
+		return ctx
+	}
+
+	isPlainUser := false
+	ugs, err := u.GroupIds()
+	if err != nil {
+		log.Warningf("looking up groups for user %s: %v", u.Username)
+	} else if u.Username == "root" {
+		// We treat root as a plain user for convenience's sake.
+		isPlainUser = true
+	} else {
+		plainUserGid := userGroupGid()
+		for _, ug := range ugs {
+			if ug == plainUserGid {
+				isPlainUser = true
+				break
+			}
+		}
+	}
+
+	return context.WithValue(ctx, userContextKey, &userData{
+		Username:    u.Username,
+		IsPlainUser: isPlainUser,
+	})
+}
+
+func main() {
+	flag.Parse()
+
+	vcfg := vapi.DefaultConfig()
+	vcfg.AgentAddress = *agentAddr
+	v, err := vapi.NewClient(vcfg)
+	if err != nil {
+		log.Exitf("creating vault client against %v: %v", *agentAddr, err)
+	}
+
+	ctx := context.Background()
+	c := newCache(&vaultTokenInteractor{v})
+	go func() {
+		t := time.NewTicker(*cacheTickInterval)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				if err := c.tick(ctx); err != nil {
+					log.Errorf("ticking the cache: %v", err)
+				}
+			}
+		}
+	}()
+	d := &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}
+	agentPath := strings.TrimPrefix(*agentAddr, "unix://")
+	vp := &vaultProxier{v: v, c: c, hc: &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				// Ignore what they want.
+				return d.DialContext(ctx, "unix", agentPath)
+			},
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}}
+
+	// Just try to delete the listen path before we start listening.
+	os.Remove(*listenPath)
+
+	listener, err := net.Listen("unix", *listenPath)
+	if err != nil {
+		log.Exitf("listening on %v: %v", *listenPath, err)
+	}
+
+	if err := os.Chmod(*listenPath, 0777); err != nil {
+		log.Exitf("chmodding our unix socket at %v: %v", *listenPath, err)
+	}
+
+	m := http.NewServeMux()
+	m.Handle("/", vp)
+	m.HandleFunc("/tokend/cachez", func(rw http.ResponseWriter, r *http.Request) {
+		c.l.RLock()
+		for username, token := range c.m {
+			accessor, _ := token.TokenSecret.TokenAccessor()
+			log.Infof("cachez: %v: accessor= %v ; expiration=%v, renewal at=%v", username, accessor, token.expiration, token.renewThreshold)
+		}
+		c.l.RUnlock()
+
+		rw.Header().Set("Content-type", "text/plain")
+		fmt.Fprintf(rw, "written to log\n")
+	})
+
+	log.Infof("listening on %v", *listenPath)
+	server := &http.Server{Handler: m, ConnContext: attachUserData}
+	log.Exit(server.Serve(listener))
+}

go/tokend/vaultissuer.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"regexp"
+	"time"
+
+	log "github.com/golang/glog"
+	vapi "github.com/hashicorp/vault/api"
+)
+
+var (
+	vaultTokenTTL       = flag.String("vault_token_ttl", "20m", "TTL at which token must be renewed.")
+	vaultTokenMaxTTL    = flag.String("vault_token_max_ttl", "1d", "TTL at which token must be reissued.")
+	vaultTokenRenewable = flag.Bool("vault_token_renewable", true, "Whether the tokens are renewable or reissued every time.")
+)
+
+type vaultTokenInteractor struct {
+	v *vapi.Client
+}
+
+var _ tokenInteractor = ((*vaultTokenInteractor)(nil))
+
+func (i *vaultTokenInteractor) Revoke(ctx context.Context, tokenSecret *TokenSecret) error {
+	// Use the token, since we have it, to revoke itself.
+	c, err := i.v.Clone()
+	if err != nil {
+		return fmt.Errorf("generating a new Vault client: %w", err)
+	}
+
+	token, err := tokenSecret.TokenID()
+	if err != nil {
+		return fmt.Errorf("getting token from secret: %w", err)
+	}
+
+	c.SetToken(token)
+
+	// Since we have the token, we could potentially use /revoke, or even /revoke-self.
+	if err := c.Auth().Token().RevokeSelf(""); err != nil {
+		return fmt.Errorf("revoking token: %w", err)
+	}
+	return nil
+}
+
+func computePolicies(selfPolicies []string, username string, isPlainUser bool) []string {
+	appMatchRE := regexp.MustCompile(fmt.Sprintf(`^(server/[^/]+/)?app/%s$`, regexp.QuoteMeta(username)))
+	userMatchRE := regexp.MustCompile(fmt.Sprintf(`^(server/[^/]+/user|server-user)(/%s)?$`, regexp.QuoteMeta(username)))
+
+	var outPolicies []string
+	for _, p := range selfPolicies {
+		if p == "default" || appMatchRE.MatchString(p) || (isPlainUser && userMatchRE.MatchString(p)) {
+			outPolicies = append(outPolicies, p)
+		}
+	}
+	return outPolicies
+}
+
+func policiesForToken(ts *TokenSecret) ([]string, error) {
+	var ps []string
+	psIntf, ok := ts.Data["policies"].([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("policies not present or not expected []interface{} type (was %T)", ts.Data["policies"])
+	}
+	for _, p := range psIntf {
+		ps = append(ps, p.(string))
+	}
+	return ps, nil
+}
+
+func (i *vaultTokenInteractor) Issue(ctx context.Context, username string, isPlainUser bool) (*TokenSecret, error) {
+	// Look up our own token to work out what policies we might want to apply.
+	// TODO: maybe consider caching this so we don't need to do it every time?
+	self, err := i.v.Auth().Token().LookupSelf()
+	if err != nil {
+		return nil, fmt.Errorf("looking up server-wide token: %w", err)
+	}
+	selfPolicies, err := policiesForToken(self)
+	if err != nil {
+		return nil, fmt.Errorf("retrieving self policies: %w", err)
+	}
+
+	wantPolicies := computePolicies(selfPolicies, username, isPlainUser)
+	log.Infof("policies for %v: %v", username, wantPolicies)
+
+	return i.v.Auth().Token().Create(&vapi.TokenCreateRequest{
+		// TODO: extend rather than replace metadata?
+		Metadata: map[string]string{
+			"app": username,
+		},
+		TTL:            *vaultTokenTTL,
+		ExplicitMaxTTL: *vaultTokenMaxTTL,
+		Renewable:      vaultTokenRenewable,
+		Policies:       wantPolicies,
+	})
+}
+
+func (i *vaultTokenInteractor) Renew(ctx context.Context, tokenSecret *TokenSecret) (*TokenSecret, error) {
+	token, err := tokenSecret.TokenID()
+	if err != nil {
+		return nil, fmt.Errorf("getting token from secret: %w", err)
+	}
+
+	return i.v.Auth().Token().RenewTokenAsSelf(token, int((20 * time.Minute).Seconds()))
+}

ops/nixos/lib/common.nix

@@ -21,6 +21,7 @@ in
     ./vault-agent-secrets.nix
     ./secretsmgr.nix
     ./secretsmgr-acme.nix
+    ./tokend.nix
     ./ssh-ca-vault.nix
   ];
 

ops/nixos/lib/rebuilder.nix

@@ -11,8 +11,8 @@ pkgs.writeShellScriptBin "rebuilder" ''
 
   DEPOT_PATH="''${1:-<depot>}"
 
-  export AWS_ACCESS_KEY_ID="$(${pkgs.vault}/bin/vault kv get --address=http://127.0.0.1:8200 -field=cacheAccessKeyID kv/apps/nix-daemon)"
+  export AWS_ACCESS_KEY_ID="$(${pkgs.vault}/bin/vault kv get --address=unix:///run/tokend/sock -field=cacheAccessKeyID kv/apps/nix-daemon)"
-  export AWS_SECRET_ACCESS_KEY="$(${pkgs.vault}/bin/vault kv get --address=http://127.0.0.1:8200 -field=cacheSecretAccessKey kv/apps/nix-daemon)"
+  export AWS_SECRET_ACCESS_KEY="$(${pkgs.vault}/bin/vault kv get --address=unix:///run/tokend/sock -field=cacheSecretAccessKey kv/apps/nix-daemon)"
 
   current_specialisation="$(cat /run/current-system/specialisation-name 2>/dev/null)"
   specialisation_path=""

ops/nixos/lib/secretsmgr.nix

@@ -95,7 +95,7 @@ in
       serviceConfig = {
         User = "secretsmgr";
         Group = "secretsmgr";
-        SupplementaryGroups = cfg.groups;
+        SupplementaryGroups = cfg.groups ++ [ "vault-agent" ];
         AmbientCapabilities = [ "CAP_SETGID" ];
 
         Type = "oneshot";

ops/nixos/lib/switch-prebuilt.nix

@@ -6,8 +6,8 @@
 pkgs.writeShellScriptBin "switch-prebuilt" ''
   set -ue
 
-  export AWS_ACCESS_KEY_ID="$(${pkgs.vault}/bin/vault kv get --address=http://127.0.0.1:8200 -field=cacheAccessKeyID kv/apps/nix-daemon)"
+  export AWS_ACCESS_KEY_ID="$(${pkgs.vault}/bin/vault kv get --address=unix:///run/tokend/sock -field=cacheAccessKeyID kv/apps/nix-daemon)"
-  export AWS_SECRET_ACCESS_KEY="$(${pkgs.vault}/bin/vault kv get --address=http://127.0.0.1:8200 -field=cacheSecretAccessKey kv/apps/nix-daemon)"
+  export AWS_SECRET_ACCESS_KEY="$(${pkgs.vault}/bin/vault kv get --address=unix:///run/tokend/sock -field=cacheSecretAccessKey kv/apps/nix-daemon)"
   system="''${1}"
 
   if [[ "$system" == "latest" ]]; then

ops/nixos/lib/tokend.nix

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{ pkgs, config, depot, lib, ... }:
+let
+  inherit (lib) mkOption types mkBefore mkIf;
+
+  cfg = config.my.vault.tokend;
+in
+{
+  options.my.vault.tokend = {
+    enable = mkOption {
+      type = types.bool;
+      default = true;
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    users.groups.tokend = {};
+    users.users.tokend = { isSystemUser = true; group = "tokend"; };
+
+    systemd.services.tokend = {
+      description = "Daemon for dynamically issuing Vault tokens based on connecting UID";
+      wants = [ "vault-agent.service" "network.target" ];
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        User = "tokend";
+        SupplementaryGroups = [ "vault-agent" ];
+        RuntimeDirectory = "tokend";
+        RuntimeDirectoryMode = "0755";
+
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = "yes";
+
+        ExecStart = "${depot.go.tokend}/bin/tokend --logtostderr";
+      };
+    };
+  };
+}

ops/nixos/lib/vault-agent.nix

@@ -56,6 +56,9 @@ in
       listener.unix = {
         address = mkDefault "/run/vault-agent/sock";
         tls_disable = mkDefault true;
+        socket_mode = "770";
+        socket_user = "vault-agent";
+        socket_group = "vault-agent";
       };
     };
 

ops/nixos/totoro/default.nix

@@ -495,6 +495,7 @@ in {
         RARITAN_PASSWORD=${secrets.raritan.sslrenew.password}
       '';
       DynamicUser = true;
+      User = "sslrenew-raritan";
       StateDirectory = "sslrenew-raritan";
       StateDirectoryMode = "0700";
       WorkingDirectory = "/var/lib/sslrenew-raritan";

ops/raritan/ssl-renew/lego.sh

@@ -6,6 +6,7 @@ CERTIFICATE_JSON="$(@curl@/bin/curl \
 	-H "X-Vault-Request: true" \
 	-X PUT \
 	-d "{\"common_name\": \"${CERTIFICATE_DOMAIN}\"}" \
+	--unix-socket "/run/tokend/sock" \
 	"http://localhost:8200/v1/acme/certs/${CERTIFICATE_ROLE}")"
 
 if [[ "$(@jq@/bin/jq .errors <(echo "$CERTIFICATE_JSON") 2>/dev/null)" != "null" ]]; then

ops/vault/cfg/config.nix

@@ -38,4 +38,14 @@
 
   my.apps.pomerium = {};
   my.servers.etheroute-lon01.apps = [ "pomerium" ];
+
+  my.apps.sslrenew-raritan = {
+    policy = ''
+      # sslrenew-raritan is permitted to issue certificates.
+      path "acme/certs/*" {
+        capabilities = ["create"]
+      }
+    '';
+  };
+  my.servers.totoro.apps = [ "sslrenew-raritan" ];
 }

ops/vault/cfg/policies-app.nix

@@ -1,7 +1,7 @@
 { lib, config, ... }:
 
 let
-  inherit (lib) mkOption types mkMerge mapAttrsToList;
+  inherit (lib) mkOption types mkMerge mapAttrsToList mkBefore;
 in {
   options.my.apps = mkOption {
     type = types.attrsOf (types.submodule ({ name, ... }: {
@@ -14,17 +14,20 @@ in {
 
         policy = mkOption {
           type = types.lines;
-          default = ''
-            path "kv/data/apps/${name}" {
-              capabilities = ["read"]
-            }
-
-            path "kv/metadata/apps/${name}" {
-              capabilities = ["read"]
-            }
-          '';
         };
       };
+
+      config = {
+        policy = mkBefore ''
+          path "kv/data/apps/${name}" {
+            capabilities = ["read"]
+          }
+
+          path "kv/metadata/apps/${name}" {
+            capabilities = ["read"]
+          }
+        '';
+      };
     }));
   };
 

ops/vault/cfg/policies/server-user.hcl

@@ -0,0 +1,10 @@
+# This policy is granted to user accounts on servers - that is, "root", and anything in the users group.
+# It allows for scoping things which shouldn't be in the Nix configuration, but are generally available to users on these machines.
+
+# "Unauthenticated" users on servers can get nix-daemon kv.
+path "kv/data/apps/nix-daemon" {
+  capabilities = ["read"]
+}
+path "kv/metadata/apps/nix-daemon" {
+  capabilities = ["read"]
+}

ops/vault/cfg/policies/server.hcl

@@ -25,3 +25,8 @@ path "kv/data/apps/nix-daemon" {
 path "kv/metadata/apps/nix-daemon" {
   capabilities = ["read"]
 }
+
+# Servers can issue sub-tokens.
+path "auth/token/create" {
+  capabilities = ["update"]
+}

ops/vault/cfg/servers.nix

@@ -28,6 +28,18 @@ let
         default = [];
       };
 
+      appPolicies = mkOption {
+        # Server-specific app policies.
+        type = with types; attrsOf lines;
+        default = {};
+      };
+
+      userPolicies = mkOption {
+        # Server-specific user policies.
+        type = with types; attrsOf lines;
+        default = {};
+      };
+
       hostnames = mkOption {
         type = with types; listOf str;
         default = [
@@ -70,7 +82,7 @@ in {
 
   config.my.servers = mapToAttrs (name: nameValuePair name {}) (builtins.attrNames depot.ops.nixos.systemConfigs);
 
-  config.resource = mkMerge (mapAttrsToList (serverName: serverCfg: {
+  config.resource = mkMerge (mapAttrsToList (serverName: serverCfg: mkMerge ([{
     vault_policy.${serverCfg.resourceName} = {
       name = "server/${serverName}";
       inherit (serverCfg) policy;
@@ -84,9 +96,11 @@ in {
       token_ttl = minutes 20;
       token_max_ttl = minutes 30;
       token_policies =
-        ["default" "server" "\${vault_policy.${serverCfg.resourceName}.name}"]
+        ["default" "server" "server-user" "\${vault_policy.${serverCfg.resourceName}.name}"]
         ++ serverCfg.extraPolicies
-        ++ (map (name: "\${vault_policy.app_${name}.name}") serverCfg.apps);
+        ++ (map (name: "\${vault_policy.app_${name}.name}") serverCfg.apps)
+        ++ (map (name: "\${vault_policy.server_${serverCfg.resourceName}_app_${name}.name}") (builtins.attrNames serverCfg.appPolicies))
+        ++ (map (name: "\${vault_policy.server_${serverCfg.resourceName}_user_${name}.name}") (builtins.attrNames serverCfg.userPolicies));
     };
 
     vault_identity_entity.${serverCfg.resourceName} = {
@@ -110,5 +124,17 @@ in {
       ttl = 7 * 24 * 60 * 60;
       max_ttl = 7 * 24 * 60 * 60;
     };
-  }) cfg);
+  }]
+  ++ mapAttrsToList (appName: policy: {
+    vault_policy.${"server_${serverCfg.resourceName}_app_${appName}"} = {
+      name = "server/${serverName}/app/${appName}";
+      inherit policy;
+    };
+  }) serverCfg.appPolicies
+  ++ mapAttrsToList (userName: policy: {
+    vault_policy.${"server_${serverCfg.resourceName}_user_${userName}"} = {
+      name = "server/${serverName}/user/${userName}";
+      inherit policy;
+    };
+  }) serverCfg.userPolicies)) cfg);
 }