depot/ops/nixos/lib/common.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ pkgs, system, config, depot, lib, rebuilder, ... }@args:
let 
  inherit (depot.ops) secrets;

  switch-prebuilt = import ./switch-prebuilt.nix args;

  # Default default priority is 1000; we use 900 here so we're higher priority,
  # but lower priority than user-specified things.  This is particularly
  # important for our timezone setting, which otherwise conflicts with the one
  # set by the Clickhouse module, which is used by e.g. bvm-logger.
  mkDefault = lib.mkOverride 900;
in
{
  imports = [
    ../../../third_party/home-manager/nixos
    ./vault-agent.nix
    ./vault-agent-secrets.nix
    ./secretsmgr.nix
    ./secretsmgr-acme.nix
    ./tokend.nix
    ./ssh-ca-vault.nix
  ];

  options.my.specialisationName = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = null;
  };
  options.my.systemType = lib.mkOption {
    type = lib.types.str;
    default = "x86_64-linux";
  };

  options.my.rundeck.hostname = lib.mkOption {
    type = lib.types.str;
    default = config.networking.fqdn;
  };
  options.my.rundeck.expectedOnline = lib.mkOption {
    type = lib.types.bool;
    default = true;
  };
  options.my.rundeck.tags = lib.mkOption {
    type = lib.types.listOf lib.types.str;
    default = [ "nixos" ];
  };

  options.my.home-manager.imports = lib.mkOption {
    type = lib.types.listOf lib.types.path;
    default = [ ./home-manager/common.nix ];
  };
  options.my.home-manager.system = lib.mkOption {
    type = lib.types.nullOr lib.types.anything;
    default = null;
  };
  options.my.prometheus.additionalExporterPorts = lib.mkOption {
    type = lib.types.attrsOf lib.types.port;
    default = {};
  };

  options.my.ip.tailscale = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = null;
  };
  options.my.ip.tailscale6 = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = null;
  };

  options.my.deploy.enable = lib.mkOption {
    type = lib.types.bool;
    default = true;
  };
  options.my.deploy.args = lib.mkOption {
    type = lib.types.str;
    default = "";
  };

  options.my.scrapeJournal.enable = lib.mkOption {
    type = lib.types.bool;
    default = config.my.scrapeJournal.addr != null;
  };
  options.my.scrapeJournal.addr = lib.mkOption {
    type = lib.types.nullOr lib.types.str;
    default = if config.my.ip.tailscale6 == null then if config.my.ip.tailscale == null then null else "${config.my.ip.tailscale}:19531" else "[${config.my.ip.tailscale6}]:19531";
  };

  config = {
    hardware.enableRedistributableFirmware = true;

    networking.search = [
      "int.as205479.net"
      "as205479.net"
    ];
    services.resolved = {
      enable = true;
      llmnr = "false";  # LLMNR breaks search domains.
      dnssec = "false";  # DNSSEC support in systemd-resolved is just broken.
      domains = config.networking.search;
      extraConfig = ''
        # For global search domains to work, we also need global DNS servers.
        DNS=8.8.8.8#dns.google [2001:4860:4860::8888]#dns.google 1.1.1.1#cloudflare-dns.com [2606:4700:4700::1111]#cloudflare-dns.com
      '';
      fallbackDns = [
        "8.8.8.8"
        "2001:4860:4860::8888"
        "1.1.1.1"
        "2606:4700:4700::1111"
      ];
    };

    my.rundeck.tags = [ "nixos" ];

    nix = {
      package = pkgs.lix;  # Use a working nix.
      nixPath = [ "depot=/home/lukegb/depot/" "nixpkgs=/home/lukegb/depot/hack/nixpkgs/" ];
      settings = {
        trusted-users = [ "root" "@wheel" "deployer" ];
        substituters = lib.mkForce [ "https://cache.nixos.org/" "s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1" ];
        trusted-substituters = lib.mkForce [ "https://cache.nixos.org/" "s3://lukegb-nix-cache?endpoint=storage.googleapis.com&trusted=1" ];
        experimental-features = "nix-command";
        max-call-depth = "9000000";
      };
    };
    nixpkgs.config = depot.third_party.nixpkgsConfig;

    documentation.nixos.enable = false;  # I just use the website.

    i18n.defaultLocale = "en_GB.UTF-8";
    console.keyMap = "us";

    time.timeZone = mkDefault "Etc/UTC";

    zramSwap = {
      enable = true;
      memoryMax = 4 * 1024 * 1024 * 1024;
    };

    environment.systemPackages = with pkgs; [
      vim rxvt-unicode-unwrapped.terminfo tmux rebuilder tailscale rsync libarchive tcpdump restic
      alacritty.terminfo kitty.terminfo
      iftop htop jq
      depot.nix.pkgs.mercurial
      switch-prebuilt
      depot.ops.vault.provision-secret-id
      wireguard-tools
    ];

    networking.useDHCP = false;
    networking.firewall = {
      allowPing = true;
      logRefusedConnections = false;
      checkReversePath = lib.mkDefault "loose";
    };

    environment.homeBinInPath = true;
    security.pam.sshAgentAuth = {
      enable = true;
      authorizedKeysFiles = [
        (toString ../../secrets/ssh-agent-pam.pub)
      ];
    };
    security.pam.ussh = {
      enable = true;
      control = "sufficient";
      caFile = toString ../../secrets/client-ca.pub;
    };

    users.mutableUsers = false;
    users.users = let secrets = depot.ops.secrets; in {
      root.hashedPassword = secrets.passwordHashes.root;
      lukegb = {
        isNormalUser = true;
        uid = 1000;
        homeMode = "711";
        extraGroups = [ "wheel" "audio" ];
        hashedPassword = secrets.passwordHashes.lukegb;
        openssh.authorizedKeys.keyFiles = [
          ../../secrets/lukegb_totoro.pub
          ../../secrets/lukegb_termius.pub
          ../../secrets/lukegb_porcorosso_win.pub
          ../../secrets/lukegb_porcorosso_wsl.pub
          ../../secrets/lukegb_porcorosso_linux.pub
          ../../secrets/lukegb_red_solo.pub
        ];
      };
      deployer = {
        isSystemUser = true;
        uid = 1001;
        group = "deployer";
        hashedPassword = "!";
        useDefaultShell = true;
        home = "/var/lib/deployer";
        createHome = true;
        openssh.authorizedKeys.keyFiles = [
          ../../secrets/deployer_ed25519.pub
          ../../secrets/rundeck_deployer_rsa.pub
        ];
      };
    };
    users.groups.deployer = {};
    security.sudo.extraRules = [{
      users = [ "deployer" ];
      commands = [{
        command = "${rebuilder}/bin/rebuilder";
        options = [ "NOPASSWD" ];
      } {
        command = "${switch-prebuilt}/bin/switch-prebuilt";
        options = [ "NOPASSWD" ];
      }];
    }];
    security.sudo.extraConfig = ''
      Defaults:deployer !requiretty
    '';

    programs.mtr.enable = true;
    services.openssh.enable = true;
    programs.ssh = {
      extraConfig = ''
        CanonicalizeHostname yes
        CanonicalDomains int.as205479.net as205479.net otter-acoustic.ts.net
        CanonicalizeMaxDots 0
        CanonicalizePermittedCNAMEs *.lukegb.com:*.as205479.net,*.int.as205479.net,*.otter-acoustic.ts.net *.lukegb.dev:*.as205479.net,*.int.as205479.net,*.otter-acoustic.ts.net *.zxcvbnm.ninja:*.as205479.net,*.int.as205479.net,*.otter-acoustic.ts.net
      '';
      knownHosts."*" = {
        certAuthority = true;
        publicKey = builtins.readFile ../../secrets/server-ca.pub;
      };
    };
    services.tailscale.enable = true;
    systemd.network.config.networkConfig.ManageForeignRoutingPolicyRules = false;
    networking.firewall.interfaces.tailscale0 = {
      # Just allow anything in on tailscale0.
      allowedTCPPortRanges = [{ from = 0; to = 65535; }];
      allowedUDPPortRanges = [{ from = 0; to = 65535; }];
    };

    boot = {
      kernelModules = [ "tcp_bbr" ];
      kernel.sysctl."net.ipv4.tcp_congestion_control" = "bbr";
      kernel.sysctl."net.core.default_qdisc" = "fq_codel";
    };

    # Clean up daily.
    nix.gc = {
      automatic = mkDefault true;
      dates = "*-*-* 05:00:00";
      options = "--delete-older-than 7d";
    };

    home-manager.useUserPackages = true;
    home-manager.useGlobalPkgs = true;

    systemd.services."home-manager-lukegb" = {
      before = [ "display-manager.service" ];
      wantedBy = [ "multi-user.target" ];
    };

    home-manager.users.lukegb = { pkgs, ... }: ({
      imports = [ ({
        _module.args = args // {
          configName = null;
        };
      })] ++ config.my.home-manager.imports ++ (
        lib.optional (config.my.home-manager.system != null) config.my.home-manager.system
      );
    });

    services.prometheus.exporters.node = {
      enable = true;
      enabledCollectors = [ "systemd" "textfile" ];
      extraFlags = [
        "--collector.textfile.directory=/run/prometheus-textfile-exports"
      ];
    };
    system.activationScripts.node-exporter = {
      text = ''
        test -d /run/prometheus-textfile-exports || mkdir /run/prometheus-textfile-exports

        my_version_string="$(cat "$systemConfig/nixos-version")"
        my_hash_string="$(readlink -f "$systemConfig" | ${pkgs.gnugrep}/bin/grep -Eo '\b[0-9a-df-np-sv-z]{32}\b')"
        my_specialisation="$(cat "$systemConfig/specialisation-name" 2>/dev/null || true)"
        echo "nixos_running_system{version=\"$my_version_string\", hash=\"$my_hash_string\", specialisation=\"$my_specialisation\"} 1" > /run/prometheus-textfile-exports/running_system.prom

        if test -e /run/booted-system; then
          my_version_string="$(cat "/run/booted-system/nixos-version")"
          my_hash_string="$(readlink -f "/run/booted-system" | ${pkgs.gnugrep}/bin/grep -Eo '\b[0-9a-df-np-sv-z]{32}\b')"
          my_specialisation="$(cat "/run/booted-system/specialisation-name" 2>/dev/null || true)"
          echo "nixos_booted_system{version=\"$my_version_string\", hash=\"$my_hash_string\", specialisation=\"$my_specialisation\"} 1" > /run/prometheus-textfile-exports/booted_system.prom
        fi

        ${depot.nix.pkgs.nixos-size}/bin/nixos-size > /run/prometheus-textfile-exports/nixos_size.prom
      '';
    };
    boot.postBootCommands = lib.mkAfter ''
      test -d /run/prometheus-textfile-exports || mkdir /run/prometheus-textfile-exports

      if test -e /run/booted-system; then
        my_version_string="$(cat "/run/booted-system/nixos-version")"
        my_hash_string="$(readlink -f "/run/booted-system" | ${pkgs.gnugrep}/bin/grep -Eo '\b[0-9a-df-np-sv-z]{32}\b')"
        my_specialisation="$(cat "/run/booted-system/specialisation-name" 2>/dev/null || true)"
        echo "nixos_booted_system{version=\"$my_version_string\", hash=\"$my_hash_string\", specialisation=\"$my_specialisation\"} 1" > /run/prometheus-textfile-exports/booted_system.prom
      fi

      ${depot.nix.pkgs.nixos-size}/bin/nixos-size > /run/prometheus-textfile-exports/nixos_size.prom
    '';
    system.extraSystemBuilderCmds = lib.mkAfter ''
      echo "${if config.my.specialisationName == null then "" else config.my.specialisationName}" > $out/specialisation-name
    '';

    system.nixos.tags = lib.mkBefore ([
      depot.version
    ] ++ lib.optional (config.my.specialisationName != null) "specialisation-${config.my.specialisationName}");

    services.nginx = {
      recommendedTlsSettings = true;
      recommendedOptimisation = true;
      recommendedGzipSettings = true;
      recommendedProxySettings = true;
    };

    my.vault.secrets.nix-daemon = {
      template = ''
        {{ with secret "kv/apps/nix-daemon" }}
        AWS_ACCESS_KEY_ID={{ .Data.data.cacheAccessKeyID }}
        AWS_SECRET_ACCESS_KEY={{ .Data.data.cacheSecretAccessKey }}
        {{ end }}
      '';
      group = "root";
      reloadOrRestartUnits = [ "nix-daemon.service" ];
    };
    # - prefix to make the file existing optional.
    systemd.services.nix-daemon.serviceConfig.EnvironmentFile = "-${config.my.vault.secrets.nix-daemon.path}";

    services.fwupd.enable = true;

    # This is enabled independently of my.scrapeJournal.enable.
    services.journald.gateway.enable = config.my.ip.tailscale != null || config.my.ip.tailscale6 != null;
    systemd.sockets.systemd-journal-gatewayd.socketConfig = lib.optionalAttrs (config.my.ip.tailscale != null) {
      ListenStream = [ "" ] ++ (lib.optional (config.my.ip.tailscale != null) "${config.my.ip.tailscale}:19531") ++ (lib.optional (config.my.ip.tailscale6 != null) "[${config.my.ip.tailscale6}:19531");
      FreeBind = true;
    };

    system.disableInstallerTools = lib.mkDefault true;

    systemd.network.wait-online = {
      anyInterface = true;
    };
  };
}