From a0400126feb6a7f6d46c3f5490df000bd4cec9a1 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sun, 9 Oct 2022 14:00:04 +0100 Subject: [PATCH] buildGo: half-ass an implementation of cgo support --- third_party/tvl/nix/buildGo/default.nix | 53 ++++++++++++++++--- .../tvl/nix/buildGo/external/default.nix | 15 ++++-- third_party/tvl/nix/buildGo/external/main.go | 44 ++++++++++----- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/third_party/tvl/nix/buildGo/default.nix b/third_party/tvl/nix/buildGo/default.nix index 8d039647f5..bca849fde2 100644 --- a/third_party/tvl/nix/buildGo/default.nix +++ b/third_party/tvl/nix/buildGo/default.nix @@ -23,7 +23,7 @@ let replaceStrings toString; - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + inherit (pkgs) lib go runCommand runCommandCC fetchFromGitHub protobuf symlinkJoin; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -40,6 +40,9 @@ let srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + anyCgo = allDeps: lib.any (d: d.cgo) allDeps; + allLDFLAGS = allDeps: lib.unique (lib.flatten (map (d: d.cgoLDFLAGS) allDeps)); + allCgoBuildInputs = allDeps: lib.unique (lib.flatten (map (d: d.cgoBuildInputs) allDeps)); xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); @@ -56,22 +59,48 @@ let # Build a Go program out of the specified files and dependencies. program = { name, srcs, deps ? [ ], x_defs ? { } }: - let uniqueDeps = allDeps (map (d: d.gopkg) deps); - in runCommand name { } '' + let + uniqueDeps = allDeps (map (d: d.gopkg) deps); + cgo = anyCgo uniqueDeps; + cgoLDFLAGS = allLDFLAGS uniqueDeps; + cgoBuildInputs = allCgoBuildInputs uniqueDeps; + runCommand = if cgo then runCommandCC else pkgs.runCommand; + in runCommand name { + buildInputs = cgoBuildInputs; + } '' ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin export GOROOT_FINAL=go - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix \ + -extldflags '${toString cgoLDFLAGS}' \ + ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. # # This outputs both the sources and compiled binary, as both are # needed when downstream packages depend on it. - package = { name, srcs, deps ? [ ], path ? name, sfiles ? [ ] }: + package = { name, srcs, deps ? [ ], path ? name, packageName ? builtins.baseNameOf path, sfiles ? [ ], cgofiles ? [ ], cfiles ? [ ], cxxfiles ? [ ], cgodeps ? [ ], cgocflags ? [ ], cgoldflags ? [ ] }: let uniqueDeps = allDeps (map (d: d.gopkg) deps); + ifCgo = do: lib.optionalString (cgofiles != [ ]) do; + cgoBuild = ifCgo '' + BUILDGO_CFLAGS="${spaceOut (lib.unique (map (d: "-I ${builtins.dirOf d}") cgofiles))} ${lib.concatStringsSep " " cgocflags}" + BUILDGO_LDFLAGS="${spaceOut cgoldflags}" + ${go}/bin/go tool cgo -trimpath=$PWD -trimpath=${go} -trimpath=$out/${path} -importpath=${path} -- $BUILDGO_CFLAGS ${spaceOut cgofiles} + for f in $PWD/_obj/*.cgo2.c $PWD/_obj/_cgo_export.c; do + cc $BUILDGO_CFLAGS -c $f -o ''${f}.o + done + cc $BUILDGO_CFLAGS -c $PWD/_obj/_cgo_main.c -o $PWD/_obj/_cgo_main.o + cc $BUILDGO_CFLAGS -o $PWD/_obj/_cgo_.o $PWD/_obj/_cgo_main.o $PWD/_obj/*.cgo2.c.o $BUILDGO_LDFLAGS + ${go}/bin/go tool cgo -dynpackage ${packageName} -dynimport $PWD/_obj/_cgo_.o -dynout $PWD/_obj/_cgo_imports.go + EXTRAGO=$PWD/_obj/*.go + ''; + cgoPack = ifCgo '' + ${go}/bin/go tool pack r $out/${path}.a $PWD/_obj/*.cgo2.c.o $PWD/_obj/_cgo_export.c.o + ''; + # The build steps below need to be executed conditionally for Go # assembly if the analyser detected any *.s files. # @@ -86,16 +115,26 @@ let ${go}/bin/go tool pack r $out/${path}.a ./asm.o ''; - gopkg = (runCommand "golib-${name}" { } '' + runCommand = if (cgofiles != [ ]) then runCommandCC else pkgs.runCommand; + + gopkg = (runCommand "golib-${name}" { + buildInputs = cgodeps; + } '' mkdir -p $out/${path} + EXTRAGO="" ${srcList path (map (s: "${s}") srcs)} ${asmBuild} - ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -trimpath=$out/${path} -p ${path} ${includeSources uniqueDeps} ${spaceOut (map (srcDest path) srcs)} + ${cgoBuild} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -trimpath=$out/${path} -p ${path} ${includeSources uniqueDeps} ${spaceOut (map (srcDest path) srcs)} $EXTRAGO ${asmPack} + ${cgoPack} '').overrideAttrs (_: { passthru = { inherit gopkg; goDeps = uniqueDeps; + cgo = cgofiles != [ ]; + cgoBuildInputs = cgodeps; + cgoLDFLAGS = cgoldflags; goImportPath = path; }; }); diff --git a/third_party/tvl/nix/buildGo/external/default.nix b/third_party/tvl/nix/buildGo/external/default.nix index f713783a58..c745507430 100644 --- a/third_party/tvl/nix/buildGo/external/default.nix +++ b/third_party/tvl/nix/buildGo/external/default.nix @@ -50,7 +50,7 @@ let last = l: elemAt l ((length l) - 1); - toPackage = self: src: path: depMap: entry: + toPackage = self: src: path: depMap: entry: cgodeps: let localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) @@ -66,7 +66,7 @@ let throw "missing foreign dependency '${d.path}' in '${path}, imported at ${d.position}'" ) depMap) - entry.foreignDeps; + (lib.filter (d: d.path != "C") entry.foreignDeps); args = { srcs = map (f: src + ("/" + f)) entry.files; @@ -77,6 +77,11 @@ let name = pathToName entry.name; path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); sfiles = map (f: src + ("/" + f)) entry.sfiles; + cgofiles = map (f: src + ("/" + f)) entry.cgofiles; + inherit cgodeps; + inherit (entry) packageName cgocflags cgoldflags; + cfiles = map (f: src + ("/" + f)) entry.cfiles; + cxxfiles = map (f: src + ("/" + f)) entry.cxxfiles; }; binArgs = args // { @@ -86,7 +91,7 @@ let if entry.isCommand then (program binArgs) else (package libArgs); in -{ src, path, deps ? [ ] }: +{ src, path, deps ? [ ], tags ? [ ], cgo ? false, cgodeps ? [ ] }: let # Build a map of dependencies (from their import paths to their # derivation) so that they can be conditionally imported only in @@ -100,10 +105,10 @@ let name = pathToName path; analysisOutput = runCommand "${name}-structure.json" { } '' - ${analyser}/bin/analyser -path ${path} -source ${src} > $out + ${analyser}/bin/analyser -path ${path} -source ${src} -tags "${lib.concatStringsSep " " tags}" ${if cgo then "-cgo" else ""} > $out ''; analysis = fromJSON (readFile analysisOutput); in lib.fix (self: foldl' lib.recursiveUpdate { } ( - map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis + map (entry: mkset entry.locator (toPackage self src path depMap entry cgodeps)) analysis )) diff --git a/third_party/tvl/nix/buildGo/external/main.go b/third_party/tvl/nix/buildGo/external/main.go index a77c43b371..7750acc016 100644 --- a/third_party/tvl/nix/buildGo/external/main.go +++ b/third_party/tvl/nix/buildGo/external/main.go @@ -30,12 +30,18 @@ var stdlibList string // and external (none-stdlib) dependencies of this package. type pkg struct { Name string `json:"name"` + PackageName string `json:"packageName"` Locator []string `json:"locator"` Files []string `json:"files"` + CgoFiles []string `json:"cgofiles"` SFiles []string `json:"sfiles"` + CFiles []string `json:"cfiles"` + CXXFiles []string `json:"cxxfiles"` LocalDeps [][]string `json:"localDeps"` ForeignDeps []foreignDep `json:"foreignDeps"` IsCommand bool `json:"isCommand"` + CgoCFLAGS []string `json:"cgocflags"` + CgoLDFLAGS []string `json:"cgoldflags"` } type foreignDep struct { @@ -84,9 +90,10 @@ func findGoDirs(at string) ([]string, error) { // analysePackage loads and analyses the imports of a single Go // package, returning the data that is required by the Nix code to // generate a derivation for this package. -func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { +func analysePackage(root, source, importpath string, stdlib map[string]bool, tags []string, cgo bool) (pkg, error) { ctx := build.Default - ctx.CgoEnabled = false + ctx.CgoEnabled = cgo + ctx.BuildTags = tags p, err := ctx.ImportDir(source, build.IgnoreVendor) if err != nil { @@ -126,24 +133,28 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk prefix = "" } - files := []string{} - for _, f := range p.GoFiles { - files = append(files, path.Join(prefix, f)) - } - - sfiles := []string{} - for _, f := range p.SFiles { - sfiles = append(sfiles, path.Join(prefix, f)) + prependPrefix := func(fs []string) []string { + out := make([]string, len(fs)) + for n, f := range fs { + out[n] = path.Join(prefix, f) + } + return out } return pkg{ Name: path.Join(importpath, prefix), + PackageName: p.Name, Locator: locator, - Files: files, - SFiles: sfiles, + Files: prependPrefix(p.GoFiles), + CgoFiles: prependPrefix(p.CgoFiles), + SFiles: prependPrefix(p.SFiles), + CFiles: prependPrefix(p.CFiles), + CXXFiles: prependPrefix(p.CXXFiles), LocalDeps: local, ForeignDeps: foreign, IsCommand: p.IsCommand(), + CgoCFLAGS: p.CgoCFLAGS, + CgoLDFLAGS: p.CgoLDFLAGS, }, nil } @@ -160,6 +171,8 @@ func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { func main() { source := flag.String("source", "", "path to directory with sources to process") path := flag.String("path", "", "import path for the package") + tagsStr := flag.String("tags", "", "tags, space-separated") + cgo := flag.Bool("cgo", false, "cgo") flag.Parse() @@ -167,6 +180,11 @@ func main() { log.Fatalf("-source flag must be specified") } + var tags []string + if len(*tagsStr) > 0 { + tags = strings.Split(*tagsStr, " ") + } + stdlibPkgs, err := loadStdlibPkgs(stdlibList) if err != nil { log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) @@ -179,7 +197,7 @@ func main() { all := []pkg{} for _, d := range goDirs { - analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + analysed, err := analysePackage(*source, d, *path, stdlibPkgs, tags, *cgo) // If the Go source analysis returned "no buildable Go files", // that directory should be skipped.