depot/third_party/nixpkgs/pkgs/stdenv/linux/default.nix

589 lines
23 KiB
Nix
Raw Normal View History

# This file constructs the standard build environment for the
# Linux platform. It's completely pure; that is, it relies on no
# external (non-Nix) tools, such as /usr/bin/gcc, and it contains a C
# compiler and linker that do not search in default locations,
# ensuring purity of components produced by it.
#
# It starts from prebuilt seed bootstrapFiles and creates a series of
# nixpkgs instances (stages) to gradually rebuild stdenv, which
# is used to build all other packages (including the bootstrapFiles).
#
# Goals of the bootstrap process:
# 1. final stdenv must not reference any of the bootstrap files.
# 2. final stdenv must not contain any of the bootstrap files
# (the only current violation is libgcc_s.so in glibc).
# 3. final stdenv must not contain any of the files directly
# generated by the bootstrap code generators (assembler, linker,
# compiler). The only current violations are: libgcc_s.so in glibc,
# the lib{mpfr,mpc,gmp,isl} which are statically linked
# into the final gcc).
#
# These goals ensure that final packages and final stdenv are built
# exclusively using nixpkgs package definitions and don't depend
# on bootstrapTools (via direct references, inclusion
# of copied code, or code compiled directly by bootstrapTools).
#
# Stages are described below along with their definitions.
#
# Debugging stdenv dependency graph:
# An useful tool to explore dependencies across stages is to use
# '__bootPackages' attribute of 'stdenv. Examples of last 3 stages:
# - stdenv
# - stdenv.__bootPackages.stdenv
# - stdenv.__bootPackages.stdenv.__bootPackages.stdenv
# - ... and so on.
#
# To explore build-time dependencies in graphical form one can use
# the following:
# $ nix-store --query --graph $(nix-instantiate -A stdenv) |
# grep -P -v '[.]sh|[.]patch|bash|[.]tar' | # avoid clutter
# dot -Tsvg > stdenv-final.svg
#
# To find all the packages built by a particular stdenv instance:
# $ for stage in 0 1 2 3 4; do
# echo "stage${stage} used in:"
# nix-store --query --graph $(nix-instantiate -A stdenv) |
# grep -P ".*bootstrap-stage${stage}-stdenv.*->.*" |
# sed 's/"[0-9a-z]\{32\}-/"/g'
# done
#
# To verify which stdenv was used to build a given final package:
# $ nix-store --query --graph $(nix-instantiate -A stdenv) |
# grep -P -v '[.]sh|[.]patch|bash|[.]tar' |
# grep -P '.*stdenv.*->.*glibc-2'
# "...-bootstrap-stage2-stdenv-linux.drv" -> "...-glibc-2.35-224.drv";
#
# For a TUI (rather than CLI) view, you can use:
#
# $ nix-tree --derivation $(nix-instantiate -A stdenv)
{ lib
, localSystem, crossSystem, config, overlays, crossOverlays ? []
, bootstrapFiles ?
let table = {
glibc = {
i686-linux = import ./bootstrap-files/i686.nix;
x86_64-linux = import ./bootstrap-files/x86_64.nix;
armv5tel-linux = import ./bootstrap-files/armv5tel.nix;
armv6l-linux = import ./bootstrap-files/armv6l.nix;
armv7l-linux = import ./bootstrap-files/armv7l.nix;
aarch64-linux = import ./bootstrap-files/aarch64.nix;
mipsel-linux = import ./bootstrap-files/loongson2f.nix;
mips64el-linux = import
(if localSystem.isMips64n32
then ./bootstrap-files/mips64el-n32.nix
else ./bootstrap-files/mips64el.nix);
powerpc64le-linux = import ./bootstrap-files/powerpc64le.nix;
riscv64-linux = import ./bootstrap-files/riscv64.nix;
};
musl = {
aarch64-linux = import ./bootstrap-files/aarch64-musl.nix;
armv6l-linux = import ./bootstrap-files/armv6l-musl.nix;
x86_64-linux = import ./bootstrap-files/x86_64-musl.nix;
};
};
# Try to find an architecture compatible with our current system. We
# just try every bootstrap weve got and test to see if it is
# compatible with or current architecture.
getCompatibleTools = lib.foldl (v: system:
if v != null then v
else if localSystem.canExecute (lib.systems.elaborate { inherit system; }) then archLookupTable.${system}
else null) null (lib.attrNames archLookupTable);
archLookupTable = table.${localSystem.libc}
or (abort "unsupported libc for the pure Linux stdenv");
files = archLookupTable.${localSystem.system} or (if getCompatibleTools != null then getCompatibleTools
else (abort "unsupported platform for the pure Linux stdenv"));
in files
}:
assert crossSystem == localSystem;
let
inherit (localSystem) system;
isFromNixpkgs = pkg: !(isFromBootstrapFiles pkg);
isFromBootstrapFiles =
pkg: pkg.passthru.isFromBootstrapFiles or false;
isBuiltByNixpkgsCompiler =
pkg: isFromNixpkgs pkg && isFromNixpkgs pkg.stdenv.cc.cc;
isBuiltByBootstrapFilesCompiler =
pkg: isFromNixpkgs pkg && isFromBootstrapFiles pkg.stdenv.cc.cc;
commonPreHook =
''
export NIX_ENFORCE_PURITY="''${NIX_ENFORCE_PURITY-1}"
export NIX_ENFORCE_NO_NATIVE="''${NIX_ENFORCE_NO_NATIVE-1}"
${lib.optionalString (system == "x86_64-linux") "NIX_LIB64_IN_SELF_RPATH=1"}
${lib.optionalString (system == "mipsel-linux") "NIX_LIB32_IN_SELF_RPATH=1"}
'';
# The bootstrap process proceeds in several steps.
# Create a standard environment by downloading pre-built binaries of
# coreutils, GCC, etc.
# Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...).
bootstrapTools = (import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) {
inherit system bootstrapFiles;
extraAttrs = lib.optionalAttrs config.contentAddressedByDefault {
__contentAddressed = true;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
};
}) // { passthru.isFromBootstrapFiles = true; };
getLibc = stage: stage.${localSystem.libc};
# This function builds the various standard environments used during
# the bootstrap. In all stages, we build an stdenv and the package
# set that can be built with that stdenv.
stageFun = prevStage:
{ name, overrides ? (self: super: {}), extraNativeBuildInputs ? [] }:
let
thisStdenv = import ../generic {
name = "${name}-stdenv-linux";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config extraNativeBuildInputs;
preHook =
''
# Don't patch #!/interpreter because it leads to retained
# dependencies on the bootstrapTools in the final stdenv.
dontPatchShebangs=1
${commonPreHook}
'';
shell = "${bootstrapTools}/bin/bash";
initialPath = [bootstrapTools];
fetchurlBoot = import ../../build-support/fetchurl/boot.nix {
inherit system;
};
cc = if prevStage.gcc-unwrapped == null
then null
else lib.makeOverridable (import ../../build-support/cc-wrapper) {
name = "${name}-gcc-wrapper";
nativeTools = false;
nativeLibc = false;
buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
inherit (prevStage) stdenv;
};
cc = prevStage.gcc-unwrapped;
bintools = prevStage.binutils;
isGNU = true;
libc = getLibc prevStage;
inherit lib;
inherit (prevStage) coreutils gnugrep;
stdenvNoCC = prevStage.ccWrapperStdenv;
};
overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; };
};
in {
inherit config overlays;
stdenv = thisStdenv;
};
in
assert bootstrapTools.passthru.isFromBootstrapFiles or false; # sanity check
[
({}: {
__raw = true;
gcc-unwrapped = null;
binutils = null;
coreutils = null;
gnugrep = null;
})
# Build a dummy stdenv with no GCC or working fetchurl. This is
# because we need a stdenv to build the GCC wrapper and fetchurl.
(prevStage: stageFun prevStage {
name = "bootstrap-stage0";
overrides = self: super: {
# We thread stage0's stdenv through under this name so downstream stages
# can use it for wrapping gcc too. This way, downstream stages don't need
# to refer to this stage directly, which violates the principle that each
# stage should only access the stage that came before it.
ccWrapperStdenv = self.stdenv;
# The Glibc include directory cannot have the same prefix as the
# GCC include directory, since GCC gets confused otherwise (it
# will search the Glibc headers before the GCC headers). So
# create a dummy Glibc here, which will be used in the stdenv of
# stage1.
${localSystem.libc} = self.stdenv.mkDerivation {
pname = "bootstrap-stage0-${localSystem.libc}";
strictDeps = true;
version = "bootstrap";
enableParallelBuilding = true;
buildCommand = ''
mkdir -p $out
ln -s ${bootstrapTools}/lib $out/lib
'' + lib.optionalString (localSystem.libc == "glibc") ''
ln -s ${bootstrapTools}/include-glibc $out/include
'' + lib.optionalString (localSystem.libc == "musl") ''
ln -s ${bootstrapTools}/include-libc $out/include
'';
passthru.isFromBootstrapFiles = true;
};
gcc-unwrapped = bootstrapTools;
binutils = import ../../build-support/bintools-wrapper {
name = "bootstrap-stage0-binutils-wrapper";
nativeTools = false;
nativeLibc = false;
buildPackages = { };
libc = getLibc self;
inherit lib;
inherit (self) stdenvNoCC coreutils gnugrep;
bintools = bootstrapTools;
};
coreutils = bootstrapTools;
gnugrep = bootstrapTools;
};
})
# Create the first "real" standard environment. This one consists
# of bootstrap tools only, and a minimal Glibc to keep the GCC
# configure script happy.
#
# For clarity, we only use the previous stage when specifying these
# stages. So stageN should only ever have references for stage{N-1}.
#
# If we ever need to use a package from more than one stage back, we
# simply re-export those packages in the middle stage(s) using the
# overrides attribute and the inherit syntax.
(prevStage:
# previous stage0 stdenv:
assert isFromBootstrapFiles prevStage.binutils.bintools;
assert isFromBootstrapFiles prevStage."${localSystem.libc}";
assert isFromBootstrapFiles prevStage.gcc-unwrapped;
assert isFromBootstrapFiles prevStage.coreutils;
assert isFromBootstrapFiles prevStage.gnugrep;
stageFun prevStage {
name = "bootstrap-stage1";
# Rebuild binutils to use from stage2 onwards.
overrides = self: super: {
binutils-unwrapped = super.binutils-unwrapped.override {
enableGold = false;
};
inherit (prevStage)
ccWrapperStdenv
gcc-unwrapped coreutils gnugrep;
${localSystem.libc} = getLibc prevStage;
# A threaded perl build needs glibc/libpthread_nonshared.a,
# which is not included in bootstrapTools, so disable threading.
# This is not an issue for the final stdenv, because this perl
# won't be included in the final stdenv and won't be exported to
# top-level pkgs as an override either.
perl = super.perl.override { enableThreading = false; enableCrypt = false; };
};
})
# 2nd stdenv that contains our own rebuilt binutils and is used for
# compiling our own Glibc.
#
(prevStage:
# previous stage1 stdenv:
assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped;
assert isFromBootstrapFiles prevStage."${localSystem.libc}";
assert isFromBootstrapFiles prevStage.gcc-unwrapped;
assert isFromBootstrapFiles prevStage.coreutils;
assert isFromBootstrapFiles prevStage.gnugrep;
stageFun prevStage {
name = "bootstrap-stage2";
overrides = self: super: {
inherit (prevStage)
ccWrapperStdenv gettext
gcc-unwrapped coreutils gnugrep
perl gnum4 bison;
dejagnu = super.dejagnu.overrideAttrs (a: { doCheck = false; } );
# We need libidn2 and its dependency libunistring as glibc dependency.
# To avoid the cycle, we build against bootstrap libc, nuke references,
# and use the result as input for our final glibc. We also pass this pair
# through, so the final package-set uses exactly the same builds.
libunistring = super.libunistring.overrideAttrs (attrs: {
postFixup = attrs.postFixup or "" + ''
${self.nukeReferences}/bin/nuke-refs "$out"/lib/lib*.so.*.*
'';
# Apparently iconv won't work with bootstrap glibc, but it will be used
# with glibc built later where we keep *this* build of libunistring,
# so we need to trick it into supporting libiconv.
env = attrs.env or {} // { am_cv_func_iconv_works = "yes"; };
});
libidn2 = super.libidn2.overrideAttrs (attrs: {
postFixup = attrs.postFixup or "" + ''
${self.nukeReferences}/bin/nuke-refs -e '${lib.getLib self.libunistring}' \
"$out"/lib/lib*.so.*.*
'';
});
# This also contains the full, dynamically linked, final Glibc.
binutils = prevStage.binutils.override {
# Rewrap the binutils with the new glibc, so both the next
# stage's wrappers use it.
libc = getLibc self;
# Unfortunately, when building gcc in the next stage, its LTO plugin
# would use the final libc but `ld` would use the bootstrap one,
# and that can fail to load. Therefore we upgrade `ld` to use newer libc;
# apparently the interpreter needs to match libc, too.
bintools = self.stdenvNoCC.mkDerivation {
pname = prevStage.bintools.bintools.pname + "-patchelfed-ld";
inherit (prevStage.bintools.bintools) version;
passthru = { inherit (prevStage.bintools.passthru) isFromBootstrapFiles; };
enableParallelBuilding = true;
dontUnpack = true;
dontBuild = true;
strictDeps = true;
# We wouldn't need to *copy* all, but it's easier and the result is temporary anyway.
installPhase = ''
mkdir -p "$out"/bin
cp -a '${prevStage.bintools.bintools}'/bin/* "$out"/bin/
chmod +w "$out"/bin/ld.bfd
patchelf --set-interpreter '${getLibc self}'/lib/ld*.so.? \
--set-rpath "${getLibc self}/lib:$(patchelf --print-rpath "$out"/bin/ld.bfd)" \
"$out"/bin/ld.bfd
'';
};
};
};
# `libtool` comes with obsolete config.sub/config.guess that don't recognize Risc-V.
extraNativeBuildInputs =
lib.optional (localSystem.isRiscV) prevStage.updateAutotoolsGnuConfigScriptsHook;
})
# Construct a third stdenv identical to the 2nd, except that this
# one uses the rebuilt Glibc from stage2. It still uses the recent
# binutils and rest of the bootstrap tools, including GCC.
(prevStage:
# previous stage2 stdenv:
assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped;
assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc};
assert isFromBootstrapFiles prevStage.gcc-unwrapped;
assert isFromBootstrapFiles prevStage.coreutils;
assert isFromBootstrapFiles prevStage.gnugrep;
stageFun prevStage {
name = "bootstrap-stage3";
overrides = self: super: rec {
inherit (prevStage)
ccWrapperStdenv
binutils coreutils gnugrep gettext
perl patchelf linuxHeaders gnum4 bison libidn2 libunistring;
${localSystem.libc} = getLibc prevStage;
gcc-unwrapped =
let makeStaticLibrariesAndMark = pkg:
lib.makeOverridable (pkg.override { stdenv = self.makeStaticLibraries self.stdenv; })
.overrideAttrs (a: { pname = "${a.pname}-stage3"; });
in super.gcc-unwrapped.override {
# Link GCC statically against GMP etc. This makes sense because
# these builds of the libraries are only used by GCC, so it
# reduces the size of the stdenv closure.
gmp = makeStaticLibrariesAndMark super.gmp;
mpfr = makeStaticLibrariesAndMark super.mpfr;
libmpc = makeStaticLibrariesAndMark super.libmpc;
isl = makeStaticLibrariesAndMark super.isl_0_20;
# Use a deterministically built compiler
# see https://github.com/NixOS/nixpkgs/issues/108475 for context
reproducibleBuild = true;
profiledCompiler = false;
};
};
extraNativeBuildInputs = [ prevStage.patchelf ] ++
# Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64.
lib.optional (!localSystem.isx86 || localSystem.libc == "musl")
prevStage.updateAutotoolsGnuConfigScriptsHook;
})
# Construct a fourth stdenv that uses the new GCC. But coreutils is
# still from the bootstrap tools.
#
(prevStage:
# previous stage3 stdenv:
assert isBuiltByBootstrapFilesCompiler prevStage.binutils-unwrapped;
assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc};
assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped;
assert isFromBootstrapFiles prevStage.coreutils;
assert isFromBootstrapFiles prevStage.gnugrep;
# Can assume prevStage.gcc-unwrapped has almost no code from
# bootstrapTools as gcc bootstraps internally. The only
# exceptions are crt files from glibc built bybootstrapTools
# used to link executables and libraries, and the
# bootstrapTools-built, statically-linked
# lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc
# (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d).
stageFun prevStage {
name = "bootstrap-stage4";
overrides = self: super: {
# Zlib has to be inherited and not rebuilt in this stage,
# because gcc (since JAR support) already depends on zlib, and
# then if we already have a zlib we want to use that for the
# other purposes (binutils and top-level pkgs) too.
inherit (prevStage) gettext gnum4 bison perl texinfo zlib linuxHeaders libidn2 libunistring;
${localSystem.libc} = getLibc prevStage;
binutils = super.binutils.override {
# Don't use stdenv's shell but our own
shell = self.bash + "/bin/bash";
# Build expand-response-params with last stage like below
buildPackages = {
inherit (prevStage) stdenv;
};
};
# force gmp to rebuild so we have the option of dynamically linking
# libgmp without creating a reference path from:
# stage5.gcc -> stage4.coreutils -> stage3.glibc -> bootstrap
gmp = lib.makeOverridable (super.gmp.override { stdenv = self.stdenv; }).overrideAttrs (a: { pname = "${a.pname}-stage4"; });
# To allow users' overrides inhibit dependencies too heavy for
# bootstrap, like guile: https://github.com/NixOS/nixpkgs/issues/181188
gnumake = super.gnumake.override { inBootstrap = true; };
gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) {
nativeTools = false;
nativeLibc = false;
isGNU = true;
buildPackages = {
inherit (prevStage) stdenv;
};
cc = prevStage.gcc-unwrapped;
bintools = self.binutils;
libc = getLibc self;
inherit lib;
inherit (self) stdenvNoCC coreutils gnugrep;
shell = self.bash + "/bin/bash";
};
};
extraNativeBuildInputs = [ prevStage.patchelf prevStage.xz ] ++
# Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64.
lib.optional (!localSystem.isx86 || localSystem.libc == "musl")
prevStage.updateAutotoolsGnuConfigScriptsHook;
})
# Construct the final stdenv. It uses the Glibc and GCC, and adds
# in a new binutils that doesn't depend on bootstrap-tools, as well
# as dynamically linked versions of all other tools.
#
# When updating stdenvLinux, make sure that the result has no
# dependency (`nix-store -qR') on bootstrapTools or the first
# binutils built.
#
(prevStage:
# previous stage4 stdenv; see stage3 comment regarding gcc,
# which applies here as well.
assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped;
assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc};
assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped;
assert isBuiltByNixpkgsCompiler prevStage.coreutils;
assert isBuiltByNixpkgsCompiler prevStage.gnugrep;
{
inherit config overlays;
stdenv = import ../generic rec {
name = "stdenv-linux";
buildPlatform = localSystem;
hostPlatform = localSystem;
targetPlatform = localSystem;
inherit config;
preHook = commonPreHook;
initialPath =
((import ../generic/common-path.nix) {pkgs = prevStage;});
extraNativeBuildInputs = [ prevStage.patchelf ] ++
# Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64.
lib.optional (!localSystem.isx86 || localSystem.libc == "musl")
prevStage.updateAutotoolsGnuConfigScriptsHook;
cc = prevStage.gcc;
shell = cc.shell;
inherit (prevStage.stdenv) fetchurlBoot;
extraAttrs = {
inherit bootstrapTools;
shellPackage = prevStage.bash;
};
disallowedRequisites = [ bootstrapTools.out ];
# Mainly avoid reference to bootstrap tools
allowedRequisites = with prevStage; with lib;
# Simple executable tools
concatMap (p: [ (getBin p) (getLib p) ]) [
gzip bzip2 xz bash binutils.bintools coreutils diffutils findutils
gawk gmp gnumake gnused gnutar gnugrep gnupatch patchelf ed file
]
# Library dependencies
++ map getLib (
[ attr acl zlib pcre libidn2 libunistring ]
++ lib.optional (gawk.libsigsegv != null) gawk.libsigsegv
)
# More complicated cases
++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] )
++ [ /*propagated from .dev*/ linuxHeaders
binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params
]
++ lib.optionals (!localSystem.isx86 || localSystem.libc == "musl")
[ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ];
overrides = self: super: {
inherit (prevStage)
gzip bzip2 xz bash coreutils diffutils findutils gawk
gnused gnutar gnugrep gnupatch patchelf
attr acl zlib pcre libunistring;
${localSystem.libc} = getLibc prevStage;
# Hack: avoid libidn2.{bin,dev} referencing bootstrap tools. There's a logical cycle.
libidn2 = import ../../development/libraries/libidn2/no-bootstrap-reference.nix {
inherit lib;
inherit (prevStage) libidn2;
inherit (self) stdenv runCommandLocal patchelf libunistring;
};
gnumake = super.gnumake.override { inBootstrap = false; };
} // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
# Need to get rid of these when cross-compiling.
inherit (prevStage) binutils binutils-unwrapped;
gcc = cc;
};
};
})
# This "no-op" stage is just a place to put the assertions about stage5.
(prevStage:
# previous stage5 stdenv; see stage3 comment regarding gcc,
# which applies here as well.
assert isBuiltByNixpkgsCompiler prevStage.binutils-unwrapped;
assert isBuiltByBootstrapFilesCompiler prevStage.${localSystem.libc};
assert isBuiltByBootstrapFilesCompiler prevStage.gcc-unwrapped;
assert isBuiltByNixpkgsCompiler prevStage.coreutils;
assert isBuiltByNixpkgsCompiler prevStage.gnugrep;
{ inherit (prevStage) config overlays stdenv; })
]