depot/nixos/tests/nebula.nix

import ./make-test-python.nix ({ pkgs, lib, ... }: let

  # We'll need to be able to trade cert files between nodes via scp.
  inherit (import ./ssh-keys.nix pkgs)
    snakeOilPrivateKey snakeOilPublicKey;

  makeNebulaNode = { config, ... }: name: extraConfig: lib.mkMerge [
    {
      # Expose nebula for doing cert signing.
      environment.systemPackages = [ pkgs.nebula ];
      users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
      services.openssh.enable = true;
      networking.firewall.enable = true; # Implicitly true, but let's make sure.
      networking.interfaces.eth1.useDHCP = false;

      services.nebula.networks.smoke = {
        # Note that these paths won't exist when the machine is first booted.
        ca = "/etc/nebula/ca.crt";
        cert = "/etc/nebula/${name}.crt";
        key = "/etc/nebula/${name}.key";
        listen = {
          host = "0.0.0.0";
          port = if (config.services.nebula.networks.smoke.isLighthouse || config.services.nebula.networks.smoke.isRelay) then 4242 else 0;
        };
      };
    }
    extraConfig
  ];

in
{
  name = "nebula";

  nodes = {

    lighthouse = { ... } @ args:
      makeNebulaNode args "lighthouse" {
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
          address = "192.168.1.1";
          prefixLength = 24;
        }];

        services.nebula.networks.smoke = {
          isLighthouse = true;
          isRelay = true;
          firewall = {
            outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
            inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
          };
        };
      };

    allowAny = { ... } @ args:
      makeNebulaNode args "allowAny" {
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
          address = "192.168.1.2";
          prefixLength = 24;
        }];

        services.nebula.networks.smoke = {
          staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
          isLighthouse = false;
          lighthouses = [ "10.0.100.1" ];
          relays = [ "10.0.100.1" ];
          firewall = {
            outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
            inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
          };
        };
      };

    allowFromLighthouse = { ... } @ args:
      makeNebulaNode args "allowFromLighthouse" {
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
          address = "192.168.1.3";
          prefixLength = 24;
        }];

        services.nebula.networks.smoke = {
          staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
          isLighthouse = false;
          lighthouses = [ "10.0.100.1" ];
          relays = [ "10.0.100.1" ];
          firewall = {
            outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
            inbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
          };
        };
      };

    allowToLighthouse = { ... } @ args:
      makeNebulaNode args "allowToLighthouse" {
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
          address = "192.168.1.4";
          prefixLength = 24;
        }];

        services.nebula.networks.smoke = {
          enable = true;
          staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
          isLighthouse = false;
          lighthouses = [ "10.0.100.1" ];
          relays = [ "10.0.100.1" ];
          firewall = {
            outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
            inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
          };
        };
      };

    disabled = { ... } @ args:
      makeNebulaNode args "disabled" {
        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
          address = "192.168.1.5";
          prefixLength = 24;
        }];

        services.nebula.networks.smoke = {
          enable = false;
          staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
          isLighthouse = false;
          lighthouses = [ "10.0.100.1" ];
          relays = [ "10.0.100.1" ];
          firewall = {
            outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
            inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
          };
        };
      };

  };

  testScript = let

    setUpPrivateKey = name: ''
      ${name}.start()
      ${name}.succeed(
          "mkdir -p /root/.ssh",
          "chmod 700 /root/.ssh",
          "cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
          "chmod 600 /root/.ssh/id_snakeoil",
          "mkdir -p /root"
      )
    '';

    # From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
    sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";

    restartAndCheckNebula = name: ip: ''
      ${name}.systemctl("restart nebula@smoke.service")
      ${name}.succeed("ping -c5 ${ip}")
    '';

    # Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
    signKeysFor = name: ip: ''
      lighthouse.wait_for_unit("sshd.service")
      ${name}.wait_for_unit("sshd.service")
      ${name}.succeed(
          "mkdir -p /etc/nebula",
          "nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
          "scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
      )
      lighthouse.succeed(
          'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
      )
      ${name}.succeed(
          "scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
          "scp ${sshOpts} root@192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
          '(id nebula-smoke >/dev/null && chown -R nebula-smoke:nebula-smoke /etc/nebula) || true'
      )
    '';

    getPublicIp = node: ''
      ${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
    '';

    # Never do this for anything security critical! (Thankfully it's just a test.)
    # Restart Nebula right after the mutual block and/or restore so the state is fresh.
    blockTrafficBetween = nodeA: nodeB: ''
      node_a = ${getPublicIp nodeA}
      node_b = ${getPublicIp nodeB}
      ${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
      ${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
      ${nodeA}.systemctl("restart nebula@smoke.service")
      ${nodeB}.systemctl("restart nebula@smoke.service")
    '';
    allowTrafficBetween = nodeA: nodeB: ''
      node_a = ${getPublicIp nodeA}
      node_b = ${getPublicIp nodeB}
      ${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
      ${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
      ${nodeA}.systemctl("restart nebula@smoke.service")
      ${nodeB}.systemctl("restart nebula@smoke.service")
    '';
  in ''
    # Create the certificate and sign the lighthouse's keys.
    ${setUpPrivateKey "lighthouse"}
    lighthouse.succeed(
        "mkdir -p /etc/nebula",
        'nebula-cert ca -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
        'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
        'chown -R nebula-smoke:nebula-smoke /etc/nebula'
    )

    # Reboot the lighthouse and verify that the nebula service comes up on boot.
    # Since rebooting takes a while, we'll just restart the service on the other nodes.
    lighthouse.shutdown()
    lighthouse.start()
    lighthouse.wait_for_unit("nebula@smoke.service")
    lighthouse.succeed("ping -c5 10.0.100.1")

    # Create keys for allowAny's nebula service and test that it comes up.
    ${setUpPrivateKey "allowAny"}
    ${signKeysFor "allowAny" "10.0.100.2/24"}
    ${restartAndCheckNebula "allowAny" "10.0.100.2"}

    # Create keys for allowFromLighthouse's nebula service and test that it comes up.
    ${setUpPrivateKey "allowFromLighthouse"}
    ${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
    ${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}

    # Create keys for allowToLighthouse's nebula service and test that it comes up.
    ${setUpPrivateKey "allowToLighthouse"}
    ${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
    ${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}

    # Create keys for disabled's nebula service and test that it does not come up.
    ${setUpPrivateKey "disabled"}
    ${signKeysFor "disabled" "10.0.100.5/24"}
    disabled.fail("systemctl status nebula@smoke.service")
    disabled.fail("ping -c5 10.0.100.5")

    # The lighthouse can ping allowAny and allowFromLighthouse but not disabled
    lighthouse.succeed("ping -c3 10.0.100.2")
    lighthouse.succeed("ping -c3 10.0.100.3")
    lighthouse.fail("ping -c3 10.0.100.5")

    # allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
    allowAny.succeed("ping -c3 10.0.100.1")
    allowAny.fail("ping -c3 10.0.100.3")

    # allowFromLighthouse can ping the lighthouse and allowAny
    allowFromLighthouse.succeed("ping -c3 10.0.100.1")
    allowFromLighthouse.succeed("ping -c3 10.0.100.2")

    # block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
    ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
    allowFromLighthouse.succeed("ping -c10 10.0.100.2")
    ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
    allowFromLighthouse.succeed("ping -c10 10.0.100.2")

    # allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
    allowToLighthouse.succeed("ping -c3 10.0.100.1")
    allowToLighthouse.fail("ping -c3 10.0.100.2")
    allowToLighthouse.fail("ping -c3 10.0.100.3")

    # allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
    allowAny.succeed("ping -c3 10.0.100.3")

    # block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
    ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
    allowFromLighthouse.succeed("ping -c10 10.0.100.2")
    allowAny.succeed("ping -c10 10.0.100.3")
    ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
    allowFromLighthouse.succeed("ping -c10 10.0.100.2")
    allowAny.succeed("ping -c10 10.0.100.3")

    # allowToLighthouse can ping allowAny if allowAny pings it first
    allowAny.succeed("ping -c3 10.0.100.4")
    allowToLighthouse.succeed("ping -c3 10.0.100.2")

    # block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
    ${blockTrafficBetween "allowAny" "allowToLighthouse"}
    allowAny.succeed("ping -c10 10.0.100.4")
    allowToLighthouse.succeed("ping -c10 10.0.100.2")
    ${allowTrafficBetween "allowAny" "allowToLighthouse"}
    allowAny.succeed("ping -c10 10.0.100.4")
    allowToLighthouse.succeed("ping -c10 10.0.100.2")

    # block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
    ${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
    ${blockTrafficBetween "allowFromLighthouse" "allowAny"}
    allowFromLighthouse.fail("ping -c3 10.0.100.2")
    ${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
    ${allowTrafficBetween "allowFromLighthouse" "allowAny"}
    allowFromLighthouse.succeed("ping -c3 10.0.100.2")

    # block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
    ${blockTrafficBetween "allowAny" "lighthouse"}
    ${blockTrafficBetween "allowAny" "allowFromLighthouse"}
    ${blockTrafficBetween "allowAny" "allowToLighthouse"}
    allowFromLighthouse.fail("ping -c3 10.0.100.2")
    allowAny.fail("ping -c3 10.0.100.3")
    allowAny.fail("ping -c3 10.0.100.4")
    ${allowTrafficBetween "allowAny" "lighthouse"}
    ${allowTrafficBetween "allowAny" "allowFromLighthouse"}
    ${allowTrafficBetween "allowAny" "allowToLighthouse"}
    allowFromLighthouse.succeed("ping -c3 10.0.100.2")
    allowAny.succeed("ping -c3 10.0.100.3")
    allowAny.succeed("ping -c3 10.0.100.4")

    # block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
    ${blockTrafficBetween "allowToLighthouse" "lighthouse"}
    ${blockTrafficBetween "allowToLighthouse" "allowAny"}
    allowAny.fail("ping -c3 10.0.100.4")
    allowToLighthouse.fail("ping -c3 10.0.100.2")
    ${allowTrafficBetween "allowToLighthouse" "lighthouse"}
    ${allowTrafficBetween "allowToLighthouse" "allowAny"}
    allowAny.succeed("ping -c3 10.0.100.4")
    allowToLighthouse.succeed("ping -c3 10.0.100.2")
  '';
})