secretsync: add

This is a helper utility for syncing filesystem files into GitLab variables, for
deploy-time secrets.

nix/pkgs/default.nix

@@ -2,4 +2,5 @@ args: {
   javaws-env = import ./javaws-env.nix args;
   plex-pass = import ./plex-pass.nix args;
   heptapod-runner = import ./heptapod-runner.nix args;
+  secretsync = import ./secretsync args;
 }

nix/pkgs/secretsync/default.nix

@@ -0,0 +1,39 @@
+{ pkgs, lib, depot, ... }:
+let
+  secretsync = pkgs.buildGoModule rec {
+    pname = "secretsync";
+    version = "0.0.1";
+
+    src = ./.;
+    modSha256 = "17m97rfxwbq7vvggvjkxrzakvlk83n0caciv80d50hgdljs3ks0m";
+
+    subPackages = [ "." ];
+
+    meta = with lib; {
+      description = "Simple package for dumping secret files from disk to GitLab variables";
+    };
+  };
+in secretsync // {
+  configure = baseConfig:
+    let
+      config = {
+        name = "secretsync";
+        pkg = secretsync;
+        gitlabAccessToken = "";
+        gitlabEndpoint = "https://hg.lukegb.com";
+        gitlabProject = "lukegb/depot";
+        variablesToFile = {};
+        logToStderr = true;
+      } // baseConfig;
+      args = {
+        gitlab_access_token = config.gitlabAccessToken;
+        gitlab_endpoint = config.gitlabEndpoint;
+        gitlab_project = config.gitlabProject;
+        variable_to_file = lib.mapAttrsToList (name: value: "${name}=${value}") config.variablesToFile;
+        logtostderr = config.logToStderr;
+      };
+    in
+    pkgs.writeShellScriptBin config.name ''
+      exec "${config.pkg}/bin/secretsync" ${lib.cli.toGNUCommandLineShell {} args}
+    '';
+}

nix/pkgs/secretsync/go.mod

@@ -0,0 +1,8 @@
+module hg.lukegb.com/lukegb/depot/nix/pkgs/secretsync
+
+go 1.14
+
+require (
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/google/go-querystring v1.0.0
+)

nix/pkgs/secretsync/go.sum

@@ -0,0 +1,4 @@
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=

nix/pkgs/secretsync/secretsync.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+
+	log "github.com/golang/glog"
+	"github.com/google/go-querystring/query"
+)
+
+var (
+	gitlabAccessToken = flag.String("gitlab_access_token", os.Getenv("GITLAB_ACCESS_TOKEN"), "GitLab API Access Token")
+	gitlabEndpoint    = flag.String("gitlab_endpoint", "https://hg.lukegb.com", "GitLab API endpoint")
+	gitlabProject     = flag.String("gitlab_project", "lukegb/depot", "Project to sync secrets into")
+
+	varToFileMap map[string]string
+)
+
+func init() {
+	flag.Var(stringMapVar{&varToFileMap}, "variable_to_file", "[VARIABLE]=[FILENAME] mapping")
+}
+
+type stringMapVar struct {
+	dst *map[string]string
+}
+
+func (v stringMapVar) String() string {
+	if v.dst == nil || *v.dst == nil {
+		return "nil"
+	}
+	return fmt.Sprintf("%#v", *v.dst)
+}
+
+func (v stringMapVar) Set(s string) error {
+	if *v.dst == nil {
+		*v.dst = make(map[string]string)
+	}
+	bits := strings.SplitN(s, "=", 2)
+	if len(bits) != 2 {
+		return fmt.Errorf("not a KEY=VALUE formatted: %v", s)
+	}
+	(*v.dst)[bits[0]] = bits[1]
+	return nil
+}
+
+// GitLabAPI encapsulates access to the GitLab API.
+type GitLabAPI struct {
+	Client      *http.Client
+	AccessToken string
+	Endpoint    string
+}
+
+func (gl *GitLabAPI) url(subpath string) (string, error) {
+	u, err := url.Parse(gl.Endpoint)
+	if err != nil {
+		return "", fmt.Errorf("parsing endpoint %q: %w", gl.Endpoint, err)
+	}
+	pv, err := url.PathUnescape(subpath)
+	if err != nil {
+		return "", fmt.Errorf("unescaping %q: %w", subpath, err)
+	}
+	u.Path = path.Join(u.Path, pv)
+	u.RawPath = path.Join(u.RawPath, subpath)
+	return u.String(), nil
+}
+
+func (gl *GitLabAPI) do(ctx context.Context, req *http.Request) (*http.Response, error) {
+	req = req.WithContext(ctx)
+	if gl.AccessToken != "" {
+		req.Header.Set("Authorization", "Bearer "+gl.AccessToken)
+	}
+	resp, err := gl.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode >= http.StatusBadRequest {
+		resp.Body.Close()
+		return nil, fmt.Errorf("%v %v failed (status %v)", req.Method, req.URL, resp.Status)
+	}
+	return resp, nil
+}
+
+func (gl *GitLabAPI) get(ctx context.Context, path string) (*http.Response, error) {
+	u, err := gl.url(path)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, fmt.Errorf("building GET request for %q: %w", u, err)
+	}
+	return gl.do(ctx, req)
+}
+
+func (gl *GitLabAPI) doWithForm(ctx context.Context, method, path string, data url.Values) (*http.Response, error) {
+	u, err := gl.url(path)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest(method, u, strings.NewReader(data.Encode()))
+	if err != nil {
+		return nil, fmt.Errorf("building %v request for %q: %w", method, u, err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	return gl.do(ctx, req)
+}
+
+// Ping just tests to see if the connection is successful by retrieving the projects endpoint.
+// The response data is ignored, but the status code is checked.
+func (gl *GitLabAPI) Ping(ctx context.Context) error {
+	resp, err := gl.get(ctx, "/api/v4/projects")
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
+type ProjectVariable struct {
+	Key          string `json:"key" url:"key"`
+	VariableType string `json:"variable_type" url:"variable_type"`
+	Value        string `json:"value" url:"value"`
+	Protected    bool   `json:"protected" url:"protected"`
+	Masked       bool   `json:"masked" url:"masked"`
+}
+
+// Variables gets the list of variables from GitLab.
+func (gl *GitLabAPI) Variables(ctx context.Context, project string) (map[string]ProjectVariable, error) {
+	resp, err := gl.get(ctx, fmt.Sprintf("/api/v4/projects/%s/variables", url.PathEscape(project)))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var pvs []ProjectVariable
+	if err := json.NewDecoder(resp.Body).Decode(&pvs); err != nil {
+		return nil, fmt.Errorf("decoding GitLab API response: %w", err)
+	}
+
+	out := make(map[string]ProjectVariable, len(pvs))
+	for _, s := range pvs {
+		out[s.Key] = s
+	}
+	return out, nil
+}
+
+// CreateVariable creates a new project variable.
+func (gl *GitLabAPI) CreateVariable(ctx context.Context, project string, varData ProjectVariable) error {
+	data, err := query.Values(varData)
+	if err != nil {
+		return fmt.Errorf("encoding variable data")
+	}
+	resp, err := gl.doWithForm(ctx, "POST", fmt.Sprintf("/api/v4/projects/%s/variables", url.PathEscape(project)), data)
+	if err != nil {
+		return err
+	}
+	if _, err := ioutil.ReadAll(resp.Body); err != nil {
+		return fmt.Errorf("reading response body to completion: %w", err)
+	}
+	resp.Body.Close()
+	return nil
+}
+
+// UpdateVariable updates a project variable.
+func (gl *GitLabAPI) UpdateVariable(ctx context.Context, project string, varData ProjectVariable) error {
+	data, err := query.Values(varData)
+	if err != nil {
+		return fmt.Errorf("encoding variable data")
+	}
+	resp, err := gl.doWithForm(ctx, "PUT", fmt.Sprintf("/api/v4/projects/%s/variables/%s", url.PathEscape(project), url.PathEscape(varData.Key)), data)
+	if err != nil {
+		return err
+	}
+	if _, err := ioutil.ReadAll(resp.Body); err != nil {
+		return fmt.Errorf("reading response body to completion: %w", err)
+	}
+	resp.Body.Close()
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	defer log.Flush()
+
+	ctx := context.Background()
+
+	gitlab := &GitLabAPI{
+		Client:      http.DefaultClient,
+		AccessToken: *gitlabAccessToken,
+		Endpoint:    *gitlabEndpoint,
+	}
+	if err := gitlab.Ping(ctx); err != nil {
+		log.Exitf("gitlab.Ping: %v", err)
+	}
+
+	vars, err := gitlab.Variables(ctx, *gitlabProject)
+	if err != nil {
+		log.Exitf("gitlab.Variables: %v", err)
+	}
+
+	for varKey, varSource := range varToFileMap {
+		varFutureDataBytes, err := ioutil.ReadFile(varSource)
+		if err != nil {
+			log.Exitf("reading secret source file %q: %v", varSource, err)
+		}
+		varFutureData := string(varFutureDataBytes)
+		varCurrent, ok := vars[varKey]
+		if ok && varCurrent.Value == varFutureData {
+			log.Infof("%q up-to-date", varKey)
+			continue
+		}
+		varFuture := ProjectVariable{
+			Key:          varKey,
+			VariableType: "file",
+			Value:        varFutureData,
+			Protected:    true,
+		}
+		if !ok {
+			log.Infof("%q needs to be created", varKey)
+			if err := gitlab.CreateVariable(ctx, *gitlabProject, varFuture); err != nil {
+				log.Errorf("CreateVariable(%q, %q): %v", *gitlabProject, varKey, err)
+			}
+		} else {
+			log.Infof("%q needs to be updated", varKey)
+			if err := gitlab.UpdateVariable(ctx, *gitlabProject, varFuture); err != nil {
+				log.Errorf("UpdateVariable(%q, %q): %v", *gitlabProject, varKey, err)
+			}
+		}
+	}
+}