depot/ops/nixos/etheroute-lon01/default.nix

# SPDX-FileCopyrightText: 2020 Luke Granger-Brown <depot@lukegb.com>
#
# SPDX-License-Identifier: Apache-2.0

{ depot, lib, pkgs, config, ... }:
{
  imports = [
    ../lib/bgp.nix
    ../lib/zfs.nix
    ../totoro/barf.nix  # eww
  ];

  boot.initrd = {
    availableKernelModules = [
      "ehci_pci"
      "ahci"
      "usbhid"
      "usb_storage"
      "sd_mod"
      "sr_mod"
      "bnx2"  # ethernet
    ];
    network = {
      enable = true;
      ssh = {
        enable = true;
        hostKeys = ["/persist/etc/ssh/ssh_host_ed25519_key"];
        authorizedKeys = map builtins.readFile config.users.users.lukegb.openssh.authorizedKeys.keyFiles;
      };
      postCommands = ''
        echo "zfs load-key -a; killall zfs" >> /root/.profile
      '';
    };
  };
  boot.kernelParams = [
    "ip=103.141.25.50::103.141.25.49:255.255.255.252:etheroute-lon01:eno1:none"
  ];
  boot.kernelModules = [ "kvm-intel" ];
  boot.kernel.sysctl = {
    "net.ipv4.conf.all.forwarding" = true;
    "net.ipv4.conf.default.forwarding" = true;
  };

  # Use the systemd-boot EFI boot loader.
  boot.loader.systemd-boot.enable = true;
  boot.loader.efi.canTouchEfiVariables = true;

  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";
  services.zfs.rollbackOnBoot = {
    enable = true;
    snapshot = "tank/local/root@blank";
  };

  systemd.services."systemd-networkd-wait-online".wantedBy = lib.mkForce [];

  fileSystems = let
    zfs = device: {
      device = device;
      fsType = "zfs";
    };
  in {
    "/" = zfs "tank/local/root";
    "/nix" = zfs "tank/local/nix";
    "/tmp" = zfs "tank/local/tmp";

    "/persist" = zfs "tank/safe/persist";
    "/home" = zfs "tank/safe/home";

    "/boot" = {
      device = "/dev/disk/by-partlabel/ESP";
      fsType = "vfat";
    };
  };

  nix.settings.max-jobs = lib.mkDefault 8;

  # Networking!
  networking = {
    hostName = "etheroute-lon01";
    domain = "as205479.net";
    hostId = "420bee1b";
    useNetworkd = true;

    nameservers = [
      "2001:4860:4860::8888"
      "2001:4860:4860::8844"
      "8.8.8.8"
      "8.8.4.4"
    ];
    defaultGateway = {
      address = "103.141.25.49";
      interface = "eno1";
    };
    defaultGateway6 = {
      address = "2a07:242:800:64::1";
      interface = "eno1";
    };
    interfaces.eno1 = {
      ipv4.addresses = [{ address = "103.141.25.50"; prefixLength = 30; }];
      ipv6.addresses = [{ address = "2a07:242:800:64::68"; prefixLength = 64; }];
    };
    #interfaces.quadv1-4 = {
    #  ipv4.addresses = [{ address = "92.118.31.254"; prefixLength = 24; }];
    #  virtual = true;
    #};
    firewall.allowedTCPPorts = [ 80 443 ];
    firewall.allowedUDPPorts = [ 51820 ];
    firewall.extraCommands = ''
      # Flush old rules.
      ip46tables -D FORWARD -j lukegb-forward 2>/dev/null || true
      for chain in lukegb-forward lukegb-fwd-accept lukegb-fwd-reject; do
        ip46tables -F "$chain" 2>/dev/null || true
        ip46tables -X "$chain" 2>/dev/null || true
      done

      ip46tables -N lukegb-fwd-accept
      ip46tables -A lukegb-fwd-accept -j ACCEPT

      ip46tables -N lukegb-fwd-reject
      ip46tables -A lukegb-fwd-reject -p tcp ! --syn -j REJECT --reject-with tcp-reset
      ip46tables -A lukegb-fwd-reject -j REJECT

      ip46tables -N lukegb-forward

      # Accept from "trusted" quadv1 interface
      ip46tables -A lukegb-forward -i quadv1 -j lukegb-fwd-accept

      # Accept to quadv1 interface if we're multipathing.
      ip46tables -A lukegb-forward -o quadv1 -j lukegb-fwd-accept

      # Accept from established/related connections.
      ip46tables -A lukegb-forward -m conntrack --ctstate ESTABLISHED,RELATED -j lukegb-fwd-accept

      # Set up the firewall.
      ip46tables -A lukegb-forward -j lukegb-fwd-reject
      ip46tables -A FORWARD -j lukegb-forward
    '';
  };
  my.ip.tailscale = "100.99.227.112";
  my.ip.tailscale6 = "fd7a:115c:a1e0:ab12:4843:cd96:6263:e370";

  systemd.network.netdevs.quadv1 = {
    netdevConfig = {
      Name = "quadv1";
      Kind = "wireguard";
    };

    wireguardConfig = {
      PrivateKeyFile = pkgs.writeText "etheroute-lon01-quadv" depot.ops.secrets.wireguard.quadv1.lukegb.privateKey;
      ListenPort = 51820;
      RouteTable = "off";
    };

    wireguardPeers = [{
      PublicKey = depot.ops.secrets.wireguard.quadv1.quadv.publicKey;
      AllowedIPs = "0.0.0.0/0,::/0";
    }];
  };
  systemd.network.networks.quadv1 = {
    matchConfig.Name = "quadv1";
    networkConfig.Address = "169.254.111.0/31";

    routes = [{
      Gateway = "169.254.111.1";
      Destination = "92.118.31.0/24";
    }];
  };

  services.openssh.hostKeys = [
    {
      path = "/persist/etc/ssh/ssh_host_ed25519_key";
      type = "ed25519";
    }
    {
      path = "/persist/etc/ssh/ssh_host_rsa_key";
      type = "rsa";
      bits = 4096;
    }
  ];

  users.users = {
    lukegb.extraGroups = [ "bird2" ];
  };

  services.lukegbgp = let local = {
    asn = 205479;
  }; in {
    enable = true;
    config = {
      local = {
        routerID = "103.141.25.50";
      };
      peering = {
        etheroute = {
          local = local // {
            v4 = "103.141.25.50";
            v6 = "2a07:242:800:64::68";
          };
          remote = {
            asn = 3170;
            export_community = 4000;
            routers = [{
              v4 = "103.141.25.49";
              v6 = "2a07:242:800:64::1";
            }];
          };
        };
        bgptoolscollector = {
          local = local // {
            v4 = "103.141.25.50";
            v6 = "2a07:242:800:64::68";
          };
          remote = {
            asn = 212232;
            export_community = 5000;
            is_route_collector = true;
            routers = [{
              v4 = "185.230.223.48";
              v6 = "2a0c:2f07:9459::b6";
              multihop = 64;
            }];
            prefix_limit.v4 = 0;
            prefix_limit.v6 = 0;
          };
        };
        quadv = {
          local = local // {
            v4 = "169.254.111.0";
          };
          remote = {
            asn = 197753;
            export_community = 4099;
            routers = [{
              v4 = "169.254.111.1";
            }];
            prefix_limit.v4 = 10;
            prefix_limit.v6 = 10;
          };
        };
      };
    };
  };

  systemd.mounts = let
    bindMount' = dir: {
      unitConfig.RequiresMountsFor = dir;
      options = "bind";
      what = "/persist${dir}";
      where = dir;
    };
    bindMountSvc = dir: svc: (bindMount' dir) // {
      bindsTo = [svc];
      partOf = [svc];
    };
    bindMountSvcDynamic = dir: svc: (bindMount' "/var/lib/private/${dir}") // {
      requiredBy = [svc];
      before = [svc];
      wantedBy = ["multi-user.target"];
    };
    bindMount = dir: (bindMount' dir) // {
      wantedBy = ["multi-user.target"];
    };
  in [
    (bindMountSvc "/var/lib/tailscale" "tailscaled.service")
    (bindMountSvcDynamic "barf-fe" "barf-fe.service")
  ];

  services.redis.servers."" = {
    enable = true;
    bind = "127.0.0.1";
  };
  services.pomerium = {
    enable = true;
    secretsFile = config.my.vault.secrets.pomerium.path;

    settings = {
      address = ":443";
      http_redirect_addr = ":80";
      dns_lookup_family = "AUTO";

      idp_provider = "google";
      idp_client_id = "136257844546-qsa6hi1oqqoq2bnt93deo4e70ggbn1p8.apps.googleusercontent.com";
      idp_request_params = {
        hd = "lukegb.com";
        login_hint = "lukegb@lukegb.com";
      };

      jwt_claims_headers = [
        "email"
        "user"
      ];

      timeout_read = "0";  # We have some long-lived connections...
      timeout_write = "0";
      timeout_idle = "0";

      forward_auth_url = "https://fwdauth.int.lukegb.com";
      authenticate_service_url = "https://auth.int.lukegb.com";
      signout_redirect_url = "https://logged-out.int.lukegb.com";

      certificates = [
        { cert = "/var/lib/acme/lukegb.com/fullchain.pem"; key = "/var/lib/acme/lukegb.com/privkey.pem"; }
      ];

      policy = let
        baseConfig = {
          allowed_domains = [ "lukegb.com" ];
          pass_identity_headers = true;
          timeout = "30s";
        };
        service = server: hostName: extraConfig: baseConfig // {
          from = "https://${hostName}";
          to = "http://${server}";
          preserve_host_header = true;
        } // extraConfig;
        secureService = server: hostName: extraConfig: service server hostName ({
          to = "https://${server}";
          tls_server_name = hostName;
        } // extraConfig);
        public = extraConfig: {
          allow_public_unauthenticated_access = true;
          allowed_domains = null;
        } // extraConfig;
      in [
        (service "localhost:12001" "barf.lukegb.com" (public {}))
        (service "cofractal-ams01.int.as205479.net" "int.lukegb.com" {})
        (service "cofractal-ams01.int.as205479.net" "logged-out.int.lukegb.com" (public {}))
        (service "cofractal-ams01.int.as205479.net" "sonarr.int.lukegb.com" {})
        (service "cofractal-ams01.int.as205479.net" "radarr.int.lukegb.com" {})
        (service "cofractal-ams01.int.as205479.net" "deluge.int.lukegb.com" {})
        (service "cofractal-ams01.int.as205479.net" "content.int.lukegb.com" (public {
          timeout = "0";  # Downloads can take a while; bump the timeout.
        }))
        (service "totoro.int.as205479.net:9090" "prometheus.int.lukegb.com" {})
        (service "totoro.int.as205479.net:9093" "alertmanager.int.lukegb.com" {})
        (service "totoro.int.as205479.net:3000" "grafana.int.lukegb.com" {})
        (service "totoro.int.as205479.net:10908" "tumblrandom.int.lukegb.com" {})
        (secureService "swann.int.as205479.net:8443" "unifi.int.lukegb.com" {
          tls_skip_verify = true;
          allow_websockets = true;
          timeout = "0";
        })
        (service "blade-tuvok.int.as205479.net:7480" "objdump.zxcvbnm.ninja" (public {
          timeout = "30m";  # Uploads can take a while; bump the timeout.
        }))
        (secureService "totoro.int.as205479.net" "invoices.lukegb.com" (public {
          regex = "^/((third_party|ajax|client_area|pdf)/.*|[a-zA-Z0-9]{8})$";
          tls_skip_verify = true;
        }))
        (secureService "totoro.int.as205479.net" "invoices.lukegb.com" {
          tls_skip_verify = true;
        })
        (baseConfig // {
          from = "https://httpbin.int.lukegb.com";
          to = "https://verify.pomerium.com";
        })
        (service "bvm-twitterchiver.int.as205479.net:8080" "twitterchiver.int.lukegb.com" {})
        (service "bvm-twitterchiver.int.as205479.net:8080" "twitterchiver.lukegb.com" {})
        (service "bvm-nixosmgmt.int.as205479.net:4440" "rundeck.int.lukegb.com" {
          set_request_headers = {
            "X-Forwarded-Roles" = "pomerium";
          };
        })
        (service "bvm-ipfs.int.as205479.net:5001" "ipfs.int.lukegb.com" {})
        (service "bvm-ipfs.int.as205479.net:8080" "ipfs-gw.int.lukegb.com" {})
        (service "bvm-netbox.int.as205479.net:80" "netbox.int.lukegb.com" {})
        (service "localhost:9901" "envoy-debug.int.lukegb.com" {})
        (service "bvm-paperless.int.as205479.net:28981" "paperless.int.lukegb.com" {
          regex = "^/ws/.*";
          allow_websockets = true;
          timeout = "0";
        })
        (service "bvm-paperless.int.as205479.net:28981" "paperless.int.lukegb.com" {
          regex = "^/api/.*";
          timeout = "0";
        })
        (service "bvm-paperless.int.as205479.net:28981" "paperless.int.lukegb.com" {})
        (service "totoro.int.as205479.net:1880" "nodered.int.lukegb.com" {
          allow_websockets = true;
          timeout = "0";
        })
        (service "totoro.int.as205479.net:3002" "code.int.lukegb.com" {
          allow_websockets = true;
          timeout = "0";
        })
        (service "totoro.int.as205479.net:8099" "zigbee2mqtt.int.lukegb.com" {
          allow_websockets = true;
          timeout = "0";
        })
      ];
    };
  };
  systemd.services.pomerium = {
    wants = lib.mkAfter [ "redis.service" ];
    after = lib.mkAfter [ "redis.service" ];
    serviceConfig = {
      SupplementaryGroups = [ "acme" ];
      ReadOnlyPaths = [ "/var/lib/acme" ];
    };
  };
  my.vault.acmeCertificates."lukegb.com" = {
    hostnames = [
      "lukegb.com"
      "*.lukegb.com"
      "*.int.lukegb.com"
      "objdump.zxcvbnm.ninja"
    ];
    reloadOrRestartUnits = [ "pomerium.service" ];
  };
  my.vault.secrets.pomerium = {
    template = ''
      {{ with secret "kv/apps/pomerium" }}
      COOKIE_SECRET={{ .Data.data.cookieSecret }}
      SHARED_SECRET={{ .Data.data.sharedSecret }}
      IDP_CLIENT_SECRET={{ .Data.data.idpClientSecret }}
      SIGNING_KEY={{ .Data.data.signingKey }}
      IDP_SERVICE_ACCOUNT={{ .Data.data.googleServiceAccount }}
      {{ end }}
    '';
    group = "root";
    reloadOrRestartUnits = [ "pomerium.service" ];
  };
  users.groups.acme = {};

  system.stateVersion = "22.11";
}