diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 1f6b17b38e..dbd6a0a25c 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -2,4 +2,5 @@ args: { javaws-env = import ./javaws-env.nix args; plex-pass = import ./plex-pass.nix args; heptapod-runner = import ./heptapod-runner.nix args; + secretsync = import ./secretsync args; } diff --git a/nix/pkgs/secretsync/default.nix b/nix/pkgs/secretsync/default.nix new file mode 100644 index 0000000000..40df758461 --- /dev/null +++ b/nix/pkgs/secretsync/default.nix @@ -0,0 +1,39 @@ +{ pkgs, lib, depot, ... }: +let + secretsync = pkgs.buildGoModule rec { + pname = "secretsync"; + version = "0.0.1"; + + src = ./.; + modSha256 = "17m97rfxwbq7vvggvjkxrzakvlk83n0caciv80d50hgdljs3ks0m"; + + subPackages = [ "." ]; + + meta = with lib; { + description = "Simple package for dumping secret files from disk to GitLab variables"; + }; + }; +in secretsync // { + configure = baseConfig: + let + config = { + name = "secretsync"; + pkg = secretsync; + gitlabAccessToken = ""; + gitlabEndpoint = "https://hg.lukegb.com"; + gitlabProject = "lukegb/depot"; + variablesToFile = {}; + logToStderr = true; + } // baseConfig; + args = { + gitlab_access_token = config.gitlabAccessToken; + gitlab_endpoint = config.gitlabEndpoint; + gitlab_project = config.gitlabProject; + variable_to_file = lib.mapAttrsToList (name: value: "${name}=${value}") config.variablesToFile; + logtostderr = config.logToStderr; + }; + in + pkgs.writeShellScriptBin config.name '' + exec "${config.pkg}/bin/secretsync" ${lib.cli.toGNUCommandLineShell {} args} + ''; +} diff --git a/nix/pkgs/secretsync/go.mod b/nix/pkgs/secretsync/go.mod new file mode 100644 index 0000000000..a0d8186841 --- /dev/null +++ b/nix/pkgs/secretsync/go.mod @@ -0,0 +1,8 @@ +module hg.lukegb.com/lukegb/depot/nix/pkgs/secretsync + +go 1.14 + +require ( + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/google/go-querystring v1.0.0 +) diff --git a/nix/pkgs/secretsync/go.sum b/nix/pkgs/secretsync/go.sum new file mode 100644 index 0000000000..a218274ff0 --- /dev/null +++ b/nix/pkgs/secretsync/go.sum @@ -0,0 +1,4 @@ +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= diff --git a/nix/pkgs/secretsync/secretsync.go b/nix/pkgs/secretsync/secretsync.go new file mode 100644 index 0000000000..a0e4eae6cc --- /dev/null +++ b/nix/pkgs/secretsync/secretsync.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "strings" + + log "github.com/golang/glog" + "github.com/google/go-querystring/query" +) + +var ( + gitlabAccessToken = flag.String("gitlab_access_token", os.Getenv("GITLAB_ACCESS_TOKEN"), "GitLab API Access Token") + gitlabEndpoint = flag.String("gitlab_endpoint", "https://hg.lukegb.com", "GitLab API endpoint") + gitlabProject = flag.String("gitlab_project", "lukegb/depot", "Project to sync secrets into") + + varToFileMap map[string]string +) + +func init() { + flag.Var(stringMapVar{&varToFileMap}, "variable_to_file", "[VARIABLE]=[FILENAME] mapping") +} + +type stringMapVar struct { + dst *map[string]string +} + +func (v stringMapVar) String() string { + if v.dst == nil || *v.dst == nil { + return "nil" + } + return fmt.Sprintf("%#v", *v.dst) +} + +func (v stringMapVar) Set(s string) error { + if *v.dst == nil { + *v.dst = make(map[string]string) + } + bits := strings.SplitN(s, "=", 2) + if len(bits) != 2 { + return fmt.Errorf("not a KEY=VALUE formatted: %v", s) + } + (*v.dst)[bits[0]] = bits[1] + return nil +} + +// GitLabAPI encapsulates access to the GitLab API. +type GitLabAPI struct { + Client *http.Client + AccessToken string + Endpoint string +} + +func (gl *GitLabAPI) url(subpath string) (string, error) { + u, err := url.Parse(gl.Endpoint) + if err != nil { + return "", fmt.Errorf("parsing endpoint %q: %w", gl.Endpoint, err) + } + pv, err := url.PathUnescape(subpath) + if err != nil { + return "", fmt.Errorf("unescaping %q: %w", subpath, err) + } + u.Path = path.Join(u.Path, pv) + u.RawPath = path.Join(u.RawPath, subpath) + return u.String(), nil +} + +func (gl *GitLabAPI) do(ctx context.Context, req *http.Request) (*http.Response, error) { + req = req.WithContext(ctx) + if gl.AccessToken != "" { + req.Header.Set("Authorization", "Bearer "+gl.AccessToken) + } + resp, err := gl.Client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode >= http.StatusBadRequest { + resp.Body.Close() + return nil, fmt.Errorf("%v %v failed (status %v)", req.Method, req.URL, resp.Status) + } + return resp, nil +} + +func (gl *GitLabAPI) get(ctx context.Context, path string) (*http.Response, error) { + u, err := gl.url(path) + if err != nil { + return nil, err + } + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, fmt.Errorf("building GET request for %q: %w", u, err) + } + return gl.do(ctx, req) +} + +func (gl *GitLabAPI) doWithForm(ctx context.Context, method, path string, data url.Values) (*http.Response, error) { + u, err := gl.url(path) + if err != nil { + return nil, err + } + req, err := http.NewRequest(method, u, strings.NewReader(data.Encode())) + if err != nil { + return nil, fmt.Errorf("building %v request for %q: %w", method, u, err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return gl.do(ctx, req) +} + +// Ping just tests to see if the connection is successful by retrieving the projects endpoint. +// The response data is ignored, but the status code is checked. +func (gl *GitLabAPI) Ping(ctx context.Context) error { + resp, err := gl.get(ctx, "/api/v4/projects") + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +type ProjectVariable struct { + Key string `json:"key" url:"key"` + VariableType string `json:"variable_type" url:"variable_type"` + Value string `json:"value" url:"value"` + Protected bool `json:"protected" url:"protected"` + Masked bool `json:"masked" url:"masked"` +} + +// Variables gets the list of variables from GitLab. +func (gl *GitLabAPI) Variables(ctx context.Context, project string) (map[string]ProjectVariable, error) { + resp, err := gl.get(ctx, fmt.Sprintf("/api/v4/projects/%s/variables", url.PathEscape(project))) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var pvs []ProjectVariable + if err := json.NewDecoder(resp.Body).Decode(&pvs); err != nil { + return nil, fmt.Errorf("decoding GitLab API response: %w", err) + } + + out := make(map[string]ProjectVariable, len(pvs)) + for _, s := range pvs { + out[s.Key] = s + } + return out, nil +} + +// CreateVariable creates a new project variable. +func (gl *GitLabAPI) CreateVariable(ctx context.Context, project string, varData ProjectVariable) error { + data, err := query.Values(varData) + if err != nil { + return fmt.Errorf("encoding variable data") + } + resp, err := gl.doWithForm(ctx, "POST", fmt.Sprintf("/api/v4/projects/%s/variables", url.PathEscape(project)), data) + if err != nil { + return err + } + if _, err := ioutil.ReadAll(resp.Body); err != nil { + return fmt.Errorf("reading response body to completion: %w", err) + } + resp.Body.Close() + return nil +} + +// UpdateVariable updates a project variable. +func (gl *GitLabAPI) UpdateVariable(ctx context.Context, project string, varData ProjectVariable) error { + data, err := query.Values(varData) + if err != nil { + return fmt.Errorf("encoding variable data") + } + resp, err := gl.doWithForm(ctx, "PUT", fmt.Sprintf("/api/v4/projects/%s/variables/%s", url.PathEscape(project), url.PathEscape(varData.Key)), data) + if err != nil { + return err + } + if _, err := ioutil.ReadAll(resp.Body); err != nil { + return fmt.Errorf("reading response body to completion: %w", err) + } + resp.Body.Close() + return nil +} + +func main() { + flag.Parse() + defer log.Flush() + + ctx := context.Background() + + gitlab := &GitLabAPI{ + Client: http.DefaultClient, + AccessToken: *gitlabAccessToken, + Endpoint: *gitlabEndpoint, + } + if err := gitlab.Ping(ctx); err != nil { + log.Exitf("gitlab.Ping: %v", err) + } + + vars, err := gitlab.Variables(ctx, *gitlabProject) + if err != nil { + log.Exitf("gitlab.Variables: %v", err) + } + + for varKey, varSource := range varToFileMap { + varFutureDataBytes, err := ioutil.ReadFile(varSource) + if err != nil { + log.Exitf("reading secret source file %q: %v", varSource, err) + } + varFutureData := string(varFutureDataBytes) + varCurrent, ok := vars[varKey] + if ok && varCurrent.Value == varFutureData { + log.Infof("%q up-to-date", varKey) + continue + } + varFuture := ProjectVariable{ + Key: varKey, + VariableType: "file", + Value: varFutureData, + Protected: true, + } + if !ok { + log.Infof("%q needs to be created", varKey) + if err := gitlab.CreateVariable(ctx, *gitlabProject, varFuture); err != nil { + log.Errorf("CreateVariable(%q, %q): %v", *gitlabProject, varKey, err) + } + } else { + log.Infof("%q needs to be updated", varKey) + if err := gitlab.UpdateVariable(ctx, *gitlabProject, varFuture); err != nil { + log.Errorf("UpdateVariable(%q, %q): %v", *gitlabProject, varKey, err) + } + } + } +}