diff --git a/ci-root-linux.nix b/ci-root-linux.nix new file mode 100644 index 0000000000..2c80e5cdb5 --- /dev/null +++ b/ci-root-linux.nix @@ -0,0 +1,18 @@ +let + depot = (import ./default.nix { }); + pkgs = depot.third_party.nixpkgs; + inherit (pkgs) lib; + + arches = [ "x86_64-linux" "aarch64-linux" ]; + archRoots = builtins.listToAttrs (map (name: { inherit name; value = (import ./ci-root.nix { system = name; }); }) arches); + combined = pkgs.runCommand "systems.json" { + nativeBuildInputs = [ pkgs.jq ]; + } '' + jq -s add ${builtins.concatStringsSep " " (lib.mapAttrsToList (name: value: "'${value}/other-systemPathJSON'") archRoots)} > $out + ''; +in + pkgs.linkFarm "ci-linux-combined" [ + { name = "x86_64-linux"; path = (import ./ci-root.nix { system = "x86_64-linux"; }); } + { name = "aarch64-linux"; path = (import ./ci-root.nix { system = "aarch64-linux"; }); } + { name = "combined-systems"; path = combined; } + ] diff --git a/nix/gitlab-ci/default.nix b/nix/gitlab-ci/default.nix index a7c14d81ef..109c4f5120 100644 --- a/nix/gitlab-ci/default.nix +++ b/nix/gitlab-ci/default.nix @@ -13,18 +13,35 @@ let "nix build -v -f ./ci-root.nix --system ${system} --argstr system ${system} --substituters \"https://cache.nixos.org/ s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1\"" "nix copy -v --to 's3://lukegb-nix-cache?endpoint=storage.googleapis.com' ./result" ]; + timeout = "6h"; allow_failure = true; tags = [ "macos" ]; }; - linux = system: { + linux-eval = { stage = "build"; image = "nixos/nix:latest"; script = [ "nix run -f ./ third_party.nixpkgs.bash -c ./hack/populate_secrets.sh" - "nix build -v -f ./ci-root.nix --system ${system} --argstr system ${system} --substituters \"https://cache.nixos.org/ s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1\"" + "nix-instantiate ./ci-root-linux.nix --option substituters \"https://cache.nixos.org/ s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1\" > drv-name" + "ln -s $(cat drv-name) ./result" "nix run -f ./ go.nix.bcacheup -c bcacheup --cache_url vaultgs://lukegb-nix-cache --vault_addr unix:///run/tokend/sock --vault_token_source gcp/roleset/binary-cache-deployer/token ./result" - "cat ./result/other-systemPathJSON > systems.json" ]; + artifacts = { + paths = [ "drv-name" ]; + expire_in = "30 days"; + }; + tags = [ "cacher" ]; + }; + linux-combined = { + stage = "build"; + image = "nixos/nix:latest"; + needs = [{ job = "nixCache-linux-eval"; artifacts = true; }]; + script = [ + "nix-store --realise $(cat drv-name) --option substituters \"https://cache.nixos.org/ s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1\"" + "nix run -f ./ go.nix.bcacheup -c bcacheup --cache_url vaultgs://lukegb-nix-cache --vault_addr unix:///run/tokend/sock --vault_token_source gcp/roleset/binary-cache-deployer/token ./result" + "cat ./result/combined-systems > systems.json" + ]; + timeout = "6h"; artifacts = { paths = [ "systems.json" ]; expire_in = "30 days"; @@ -34,14 +51,14 @@ let in { stages = [ "build" "deploy-mach" "deploy-other" ]; - nixCache-x86_64-linux = linux "x86_64-linux"; - nixCache-aarch64-linux = linux "aarch64-linux"; + nixCache-linux-eval = linux-eval; + nixCache-linux = linux-combined; nixCache-x86_64-darwin = macOS "x86_64-darwin"; nixCache-aarch64-darwin = macOS "aarch64-darwin"; flipperzero-firmware = { stage = "deploy-other"; - needs = [{ job = "nixCache-x86_64-linux"; artifacts = false; }]; + needs = [{ job = "nixCache-linux"; artifacts = false; }]; tags = [ "cacher" ]; only.refs = [ "branch/default" ]; @@ -52,7 +69,7 @@ let }; lukegbcom = { stage = "deploy-other"; - needs = [{ job = "nixCache-x86_64-linux"; artifacts = false; }]; + needs = [{ job = "nixCache-linux"; artifacts = false; }]; tags = [ "cacher" ]; only.refs = [ "branch/default" ]; @@ -67,7 +84,7 @@ let deployMachs = lib.filterAttrs (name: cfg: cfg.config.my.deploy.enable) depot.ops.nixos.systemConfigs; deployStage = machName: mach: ({ stage = "deploy-mach"; - needs = [{ job = "nixCache-${mach.config.my.systemType}"; artifacts = true; }]; + needs = [{ job = "nixCache-linux"; artifacts = true; }]; tags = [ "deployer" ]; resource_group = machName;