From 786b4216c5481d8826c42defabed4721a74e1cd0 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sat, 18 Sep 2021 02:55:10 +0000 Subject: [PATCH 1/4] gn1924: init at 2021-08-08, use generic derivation generator Split into "current" version, as used by most things (aka gn), and "gn1924", which uses a more recent version of gn which is incompatible with the currently packaged version of v8 in nixpkgs. We can't win, but I need a newer version of gn for envoy. Note that the newer gn matches the version in Chromium's DEPS for v93.0.4577.82, the current Linux stable build as of September. --- .../tools/build-managers/gn/default.nix | 58 +----------------- .../tools/build-managers/gn/generic.nix | 60 +++++++++++++++++++ .../tools/build-managers/gn/rev1924.nix | 8 +++ 3 files changed, 70 insertions(+), 56 deletions(-) create mode 100644 pkgs/development/tools/build-managers/gn/generic.nix create mode 100644 pkgs/development/tools/build-managers/gn/rev1924.nix diff --git a/pkgs/development/tools/build-managers/gn/default.nix b/pkgs/development/tools/build-managers/gn/default.nix index 3c0abb3edeab5..508a821d74950 100644 --- a/pkgs/development/tools/build-managers/gn/default.nix +++ b/pkgs/development/tools/build-managers/gn/default.nix @@ -1,64 +1,10 @@ -{ stdenv, lib, fetchgit, darwin, writeText -, ninja, python3 -}: +{ callPackage, ... } @ args: -let +callPackage ./generic.nix args { # Note: Please use the recommended version for Chromium, e.g.: # https://git.archlinux.org/svntogit/packages.git/tree/trunk/chromium-gn-version.sh?h=packages/gn rev = "fd3d768bcfd44a8d9639fe278581bd9851d0ce3a"; revNum = "1718"; # git describe HEAD --match initial-commit | cut -d- -f3 version = "2020-03-09"; sha256 = "1asc14y8by7qcn10vbk467hvx93s30pif8r0brissl0sihsaqazr"; - - revShort = builtins.substring 0 7 rev; - lastCommitPosition = writeText "last_commit_position.h" '' - #ifndef OUT_LAST_COMMIT_POSITION_H_ - #define OUT_LAST_COMMIT_POSITION_H_ - - #define LAST_COMMIT_POSITION_NUM ${revNum} - #define LAST_COMMIT_POSITION "${revNum} (${revShort})" - - #endif // OUT_LAST_COMMIT_POSITION_H_ - ''; - -in stdenv.mkDerivation { - pname = "gn-unstable"; - inherit version; - - src = fetchgit { - # Note: The TAR-Archives (+archive/${rev}.tar.gz) are not deterministic! - url = "https://gn.googlesource.com/gn"; - inherit rev sha256; - }; - - nativeBuildInputs = [ ninja python3 ]; - buildInputs = lib.optionals stdenv.isDarwin (with darwin; with apple_sdk.frameworks; [ - libobjc - cctools - - # frameworks - ApplicationServices - Foundation - AppKit - ]); - - buildPhase = '' - python build/gen.py --no-last-commit-position - ln -s ${lastCommitPosition} out/last_commit_position.h - ninja -j $NIX_BUILD_CORES -C out gn - ''; - - installPhase = '' - install -vD out/gn "$out/bin/gn" - ''; - - setupHook = ./setup-hook.sh; - - meta = with lib; { - description = "A meta-build system that generates build files for Ninja"; - homepage = "https://gn.googlesource.com/gn"; - license = licenses.bsd3; - platforms = platforms.unix; - maintainers = with maintainers; [ stesie matthewbauer primeos ]; - }; } diff --git a/pkgs/development/tools/build-managers/gn/generic.nix b/pkgs/development/tools/build-managers/gn/generic.nix new file mode 100644 index 0000000000000..4214bb822b994 --- /dev/null +++ b/pkgs/development/tools/build-managers/gn/generic.nix @@ -0,0 +1,60 @@ +{ stdenv, lib, fetchgit, darwin, writeText +, ninja, python3 +, ... +}: + +{ rev, revNum, version, sha256 }: + +let + revShort = builtins.substring 0 7 rev; + lastCommitPosition = writeText "last_commit_position.h" '' + #ifndef OUT_LAST_COMMIT_POSITION_H_ + #define OUT_LAST_COMMIT_POSITION_H_ + + #define LAST_COMMIT_POSITION_NUM ${revNum} + #define LAST_COMMIT_POSITION "${revNum} (${revShort})" + + #endif // OUT_LAST_COMMIT_POSITION_H_ + ''; + +in stdenv.mkDerivation { + pname = "gn-unstable"; + inherit version; + + src = fetchgit { + # Note: The TAR-Archives (+archive/${rev}.tar.gz) are not deterministic! + url = "https://gn.googlesource.com/gn"; + inherit rev sha256; + }; + + nativeBuildInputs = [ ninja python3 ]; + buildInputs = lib.optionals stdenv.isDarwin (with darwin; with apple_sdk.frameworks; [ + libobjc + cctools + + # frameworks + ApplicationServices + Foundation + AppKit + ]); + + buildPhase = '' + python build/gen.py --no-last-commit-position + ln -s ${lastCommitPosition} out/last_commit_position.h + ninja -j $NIX_BUILD_CORES -C out gn + ''; + + installPhase = '' + install -vD out/gn "$out/bin/gn" + ''; + + setupHook = ./setup-hook.sh; + + meta = with lib; { + description = "A meta-build system that generates build files for Ninja"; + homepage = "https://gn.googlesource.com/gn"; + license = licenses.bsd3; + platforms = platforms.unix; + maintainers = with maintainers; [ stesie matthewbauer primeos ]; + }; +} diff --git a/pkgs/development/tools/build-managers/gn/rev1924.nix b/pkgs/development/tools/build-managers/gn/rev1924.nix new file mode 100644 index 0000000000000..1b17328f2e095 --- /dev/null +++ b/pkgs/development/tools/build-managers/gn/rev1924.nix @@ -0,0 +1,8 @@ +{ callPackage, ... } @ args: + +callPackage ./generic.nix args { + rev = "24e2f7df92641de0351a96096fb2c490b2436bb8"; + revNum = "1924"; # git describe HEAD --match initial-commit | cut -d- -f3 + version = "2021-08-08"; + sha256 = "1lwkyhfhw0zd7daqz466n7x5cddf0danr799h4jg3s0yvd4galjl"; +} From 637d735ad55d3d69bab6a4360327db8f988b86bb Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sat, 18 Sep 2021 02:56:17 +0000 Subject: [PATCH 2/4] envoy: 1.17.3 -> 1.19.1 This now uses gn1924 to allow v8 to build properly. --- pkgs/servers/http/envoy/default.nix | 14 ++++---------- pkgs/top-level/all-packages.nix | 2 ++ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pkgs/servers/http/envoy/default.nix b/pkgs/servers/http/envoy/default.nix index d26782560a470..c81d79dbb24be 100644 --- a/pkgs/servers/http/envoy/default.nix +++ b/pkgs/servers/http/envoy/default.nix @@ -17,8 +17,8 @@ let # However, the version string is more useful for end-users. # These are contained in a attrset of their own to make it obvious that # people should update both. - version = "1.17.3"; - commit = "46bf743b97d0d3f01ff437b2f10cc0bd9cdfe6e4"; + version = "1.19.1"; + commit = "a2a1e3eed4214a38608ec223859fcfa8fb679b14"; }; in buildBazelPackage rec { @@ -28,7 +28,7 @@ buildBazelPackage rec { owner = "envoyproxy"; repo = "envoy"; rev = srcVer.commit; - hash = "sha256:09zzr4h3zjsb2rkxrvlazpx0jy33yn9j65ilxiqbvv0ckaralqfc"; + hash = "sha256:1v1hv4blrppnhllsxd9d3k2wl6nhd59r4ydljy389na3bb41jwf9"; extraPostFetch = '' chmod -R +w $out @@ -58,7 +58,7 @@ buildBazelPackage rec { ]; fetchAttrs = { - sha256 = "sha256:1cy2b73x8jzczq9z9c1kl7zrg5iasvsakb50zxn4mswpmajkbj5h"; + sha256 = "sha256:0vnl0gq6nhvyzz39jg1bvvna0xyhxalg71bp1jbxib7ql026004r"; dontUseCmakeConfigure = true; dontUseGnConfigure = true; preInstall = '' @@ -75,12 +75,6 @@ buildBazelPackage rec { $bazelOut/external/local_config_sh/BUILD rm -r $bazelOut/external/go_sdk - # Replace some wheels which are only used for tests with empty files; - # they're nondeterministically built and packed. - >$bazelOut/external/config_validation_pip3/PyYAML-5.3.1-cp38-cp38-linux_x86_64.whl - >$bazelOut/external/protodoc_pip3/PyYAML-5.3.1-cp38-cp38-linux_x86_64.whl - >$bazelOut/external/thrift_pip3/thrift-0.13.0-cp38-cp38-linux_x86_64.whl - # Remove Unix timestamps from go cache. rm -rf $bazelOut/external/bazel_gazelle_go_repository_cache/{gocache,pkg/mod/cache,pkg/sumdb} ''; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 542235a61f109..3cfdd5f4edb85 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -14956,6 +14956,7 @@ with pkgs; nimbo = with python3Packages; callPackage ../applications/misc/nimbo { }; gn = callPackage ../development/tools/build-managers/gn { }; + gn1924 = callPackage ../development/tools/build-managers/gn/rev1924.nix { }; nixbang = callPackage ../development/tools/misc/nixbang { pythonPackages = python3Packages; @@ -20738,6 +20739,7 @@ with pkgs; envoy = callPackage ../servers/http/envoy { go = go_1_15; jdk = openjdk11; + gn = gn1924; }; etcd = callPackage ../servers/etcd { }; From 4099f938597110708889eed18e81511fdfecc1db Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Sat, 18 Sep 2021 02:57:32 +0000 Subject: [PATCH 3/4] pomerium: 0.14.7 -> 0.15.7 --- pkgs/servers/http/pomerium/default.nix | 39 +++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/pkgs/servers/http/pomerium/default.nix b/pkgs/servers/http/pomerium/default.nix index 7b28200b284e6..9f24d64ae6ca8 100644 --- a/pkgs/servers/http/pomerium/default.nix +++ b/pkgs/servers/http/pomerium/default.nix @@ -11,15 +11,15 @@ let in buildGoModule rec { pname = "pomerium"; - version = "0.14.7"; + version = "0.15.7"; src = fetchFromGitHub { owner = "pomerium"; repo = "pomerium"; rev = "v${version}"; - hash = "sha256:1jb96jk5qmary4fi1z9zwmppdyskj0qb6qii8s8mwazjjxqj1z2s"; + hash = "sha256:0adlk4ylny1z43x1dw3ny0s1932vhb61hpf5wdz4r65y8k9qyfgr"; }; - vendorSha256 = "sha256:1daabi9qc9nx8bafn26iw6rv4vx2xpd0nnk06265aqaksx26db0s"; + vendorSha256 = "sha256:1fszfbra84pcs8v1h2kf7iy603vf9v2ysg6il76aqmqrxmb1p7nv"; subPackages = [ "cmd/pomerium" "cmd/pomerium-cli" @@ -38,24 +38,25 @@ buildGoModule rec { "${varFlags}" ]; - nativeBuildInputs = [ - zip - ]; + preBuild = '' + rm internal/envoy/files/files_{darwin,linux}*.go + cat <internal/envoy/files/files_generic.go + package files + + import _ "embed" // embed + + //go:embed envoy + var rawBinary []byte - # Pomerium expects to have envoy append to it in a zip. - # We use a store-only (-0) zip, so that the Nix scanner can find any store references we had in the envoy binary. - postBuild = '' - # Append Envoy - pushd $NIX_BUILD_TOP - mkdir -p envoy - cd envoy - cp ${envoy}/bin/envoy envoy - zip -0 envoy.zip envoy - popd + //go:embed envoy.sha256 + var rawChecksum string - mv $GOPATH/bin/pomerium $GOPATH/bin/pomerium.old - cat $GOPATH/bin/pomerium.old $NIX_BUILD_TOP/envoy/envoy.zip >$GOPATH/bin/pomerium - zip --adjust-sfx $GOPATH/bin/pomerium + //go:embed envoy.version + var rawVersion string + EOF + cp ${envoy}/bin/envoy internal/envoy/files/envoy + sha256sum ${envoy}/bin/envoy > internal/envoy/files/envoy.sha256 + echo ${envoy.version} > internal/envoy/files/envoy.version ''; # We also need to set dontStrip to avoid having the envoy ZIP stripped off the end. From 74560e35e5c8ada70bb170be352d8996160f7be3 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Tue, 7 Dec 2021 15:04:09 +0000 Subject: [PATCH 4/4] pomerium: use on-disk envoy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can set an override path for Envoy's binary location now, so do that instead of the previous thing of embedding the binary. Note that we still need to include the SHA256/version of the binary we're referring to, but Through The Power Of Nix™ we can do that with relative ease. --- pkgs/servers/http/pomerium/default.nix | 36 ++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/pkgs/servers/http/pomerium/default.nix b/pkgs/servers/http/pomerium/default.nix index 9f24d64ae6ca8..cbf2fe1943542 100644 --- a/pkgs/servers/http/pomerium/default.nix +++ b/pkgs/servers/http/pomerium/default.nix @@ -7,7 +7,7 @@ }: let - inherit (lib) concatStringsSep mapAttrsToList; + inherit (lib) concatStringsSep concatMap id mapAttrsToList; in buildGoModule rec { pname = "pomerium"; @@ -28,24 +28,38 @@ buildGoModule rec { ldflags = let # Set a variety of useful meta variables for stamping the build with. setVars = { - Version = "v${version}"; - BuildMeta = "nixpkgs"; - ProjectName = "pomerium"; - ProjectURL = "github.com/pomerium/pomerium"; + "github.com/pomerium/pomerium/internal/version" = { + Version = "v${version}"; + BuildMeta = "nixpkgs"; + ProjectName = "pomerium"; + ProjectURL = "github.com/pomerium/pomerium"; + }; + "github.com/pomerium/pomerium/internal/envoy" = { + OverrideEnvoyPath = "${envoy}/bin/envoy"; + }; }; - varFlags = concatStringsSep " " (mapAttrsToList (name: value: "-X github.com/pomerium/pomerium/internal/version.${name}=${value}") setVars); + concatStringsSpace = list: concatStringsSep " " list; + mapAttrsToFlatList = fn: list: concatMap id (mapAttrsToList fn list); + varFlags = concatStringsSpace ( + mapAttrsToFlatList (package: packageVars: + mapAttrsToList (variable: value: + "-X ${package}.${variable}=${value}" + ) packageVars + ) setVars); in [ "${varFlags}" ]; preBuild = '' + # Replace embedded envoy with nothing. + # We set OverrideEnvoyPath above, so rawBinary should never get looked at + # but we still need to set a checksum/version. rm internal/envoy/files/files_{darwin,linux}*.go cat <internal/envoy/files/files_generic.go package files import _ "embed" // embed - //go:embed envoy var rawBinary []byte //go:embed envoy.sha256 @@ -54,14 +68,10 @@ buildGoModule rec { //go:embed envoy.version var rawVersion string EOF - cp ${envoy}/bin/envoy internal/envoy/files/envoy - sha256sum ${envoy}/bin/envoy > internal/envoy/files/envoy.sha256 - echo ${envoy.version} > internal/envoy/files/envoy.version + sha256sum '${envoy}/bin/envoy' > internal/envoy/files/envoy.sha256 + echo '${envoy.version}' > internal/envoy/files/envoy.version ''; - # We also need to set dontStrip to avoid having the envoy ZIP stripped off the end. - dontStrip = true; - installPhase = '' install -Dm0755 $GOPATH/bin/pomerium $out/bin/pomerium install -Dm0755 $GOPATH/bin/pomerium-cli $out/bin/pomerium-cli