312 lines
13 KiB
Nix
312 lines
13 KiB
Nix
import ./make-test-python.nix ({ pkgs, lib, ... }: let
|
|
|
|
# We'll need to be able to trade cert files between nodes via scp.
|
|
inherit (import ./ssh-keys.nix pkgs)
|
|
snakeOilPrivateKey snakeOilPublicKey;
|
|
|
|
makeNebulaNode = { config, ... }: name: extraConfig: lib.mkMerge [
|
|
{
|
|
# Expose nebula for doing cert signing.
|
|
environment.systemPackages = [ pkgs.nebula ];
|
|
users.users.root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
|
|
services.openssh.enable = true;
|
|
networking.firewall.enable = true; # Implicitly true, but let's make sure.
|
|
networking.interfaces.eth1.useDHCP = false;
|
|
|
|
services.nebula.networks.smoke = {
|
|
# Note that these paths won't exist when the machine is first booted.
|
|
ca = "/etc/nebula/ca.crt";
|
|
cert = "/etc/nebula/${name}.crt";
|
|
key = "/etc/nebula/${name}.key";
|
|
listen = {
|
|
host = "0.0.0.0";
|
|
port = if (config.services.nebula.networks.smoke.isLighthouse || config.services.nebula.networks.smoke.isRelay) then 4242 else 0;
|
|
};
|
|
};
|
|
}
|
|
extraConfig
|
|
];
|
|
|
|
in
|
|
{
|
|
name = "nebula";
|
|
|
|
nodes = {
|
|
|
|
lighthouse = { ... } @ args:
|
|
makeNebulaNode args "lighthouse" {
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
|
|
address = "192.168.1.1";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
services.nebula.networks.smoke = {
|
|
isLighthouse = true;
|
|
isRelay = true;
|
|
firewall = {
|
|
outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
};
|
|
};
|
|
};
|
|
|
|
allowAny = { ... } @ args:
|
|
makeNebulaNode args "allowAny" {
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
|
|
address = "192.168.1.2";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
services.nebula.networks.smoke = {
|
|
staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
|
|
isLighthouse = false;
|
|
lighthouses = [ "10.0.100.1" ];
|
|
relays = [ "10.0.100.1" ];
|
|
firewall = {
|
|
outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
};
|
|
};
|
|
};
|
|
|
|
allowFromLighthouse = { ... } @ args:
|
|
makeNebulaNode args "allowFromLighthouse" {
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
|
|
address = "192.168.1.3";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
services.nebula.networks.smoke = {
|
|
staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
|
|
isLighthouse = false;
|
|
lighthouses = [ "10.0.100.1" ];
|
|
relays = [ "10.0.100.1" ];
|
|
firewall = {
|
|
outbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
inbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
|
|
};
|
|
};
|
|
};
|
|
|
|
allowToLighthouse = { ... } @ args:
|
|
makeNebulaNode args "allowToLighthouse" {
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
|
|
address = "192.168.1.4";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
services.nebula.networks.smoke = {
|
|
enable = true;
|
|
staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
|
|
isLighthouse = false;
|
|
lighthouses = [ "10.0.100.1" ];
|
|
relays = [ "10.0.100.1" ];
|
|
firewall = {
|
|
outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
|
|
inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
};
|
|
};
|
|
};
|
|
|
|
disabled = { ... } @ args:
|
|
makeNebulaNode args "disabled" {
|
|
networking.interfaces.eth1.ipv4.addresses = lib.mkForce [{
|
|
address = "192.168.1.5";
|
|
prefixLength = 24;
|
|
}];
|
|
|
|
services.nebula.networks.smoke = {
|
|
enable = false;
|
|
staticHostMap = { "10.0.100.1" = [ "192.168.1.1:4242" ]; };
|
|
isLighthouse = false;
|
|
lighthouses = [ "10.0.100.1" ];
|
|
relays = [ "10.0.100.1" ];
|
|
firewall = {
|
|
outbound = [ { port = "any"; proto = "any"; host = "lighthouse"; } ];
|
|
inbound = [ { port = "any"; proto = "any"; host = "any"; } ];
|
|
};
|
|
};
|
|
};
|
|
|
|
};
|
|
|
|
testScript = let
|
|
|
|
setUpPrivateKey = name: ''
|
|
${name}.start()
|
|
${name}.succeed(
|
|
"mkdir -p /root/.ssh",
|
|
"chmod 700 /root/.ssh",
|
|
"cat '${snakeOilPrivateKey}' > /root/.ssh/id_snakeoil",
|
|
"chmod 600 /root/.ssh/id_snakeoil",
|
|
"mkdir -p /root"
|
|
)
|
|
'';
|
|
|
|
# From what I can tell, StrictHostKeyChecking=no is necessary for ssh to work between machines.
|
|
sshOpts = "-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oIdentityFile=/root/.ssh/id_snakeoil";
|
|
|
|
restartAndCheckNebula = name: ip: ''
|
|
${name}.systemctl("restart nebula@smoke.service")
|
|
${name}.succeed("ping -c5 ${ip}")
|
|
'';
|
|
|
|
# Create a keypair on the client node, then use the public key to sign a cert on the lighthouse.
|
|
signKeysFor = name: ip: ''
|
|
lighthouse.wait_for_unit("sshd.service")
|
|
${name}.wait_for_unit("sshd.service")
|
|
${name}.succeed(
|
|
"mkdir -p /etc/nebula",
|
|
"nebula-cert keygen -out-key /etc/nebula/${name}.key -out-pub /etc/nebula/${name}.pub",
|
|
"scp ${sshOpts} /etc/nebula/${name}.pub root@192.168.1.1:/root/${name}.pub",
|
|
)
|
|
lighthouse.succeed(
|
|
'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "${name}" -groups "${name}" -ip "${ip}" -in-pub /root/${name}.pub -out-crt /root/${name}.crt'
|
|
)
|
|
${name}.succeed(
|
|
"scp ${sshOpts} root@192.168.1.1:/root/${name}.crt /etc/nebula/${name}.crt",
|
|
"scp ${sshOpts} root@192.168.1.1:/etc/nebula/ca.crt /etc/nebula/ca.crt",
|
|
'(id nebula-smoke >/dev/null && chown -R nebula-smoke:nebula-smoke /etc/nebula) || true'
|
|
)
|
|
'';
|
|
|
|
getPublicIp = node: ''
|
|
${node}.succeed("ip --brief addr show eth1 | awk '{print $3}' | tail -n1 | cut -d/ -f1").strip()
|
|
'';
|
|
|
|
# Never do this for anything security critical! (Thankfully it's just a test.)
|
|
# Restart Nebula right after the mutual block and/or restore so the state is fresh.
|
|
blockTrafficBetween = nodeA: nodeB: ''
|
|
node_a = ${getPublicIp nodeA}
|
|
node_b = ${getPublicIp nodeB}
|
|
${nodeA}.succeed("iptables -I INPUT -s " + node_b + " -j DROP")
|
|
${nodeB}.succeed("iptables -I INPUT -s " + node_a + " -j DROP")
|
|
${nodeA}.systemctl("restart nebula@smoke.service")
|
|
${nodeB}.systemctl("restart nebula@smoke.service")
|
|
'';
|
|
allowTrafficBetween = nodeA: nodeB: ''
|
|
node_a = ${getPublicIp nodeA}
|
|
node_b = ${getPublicIp nodeB}
|
|
${nodeA}.succeed("iptables -D INPUT -s " + node_b + " -j DROP")
|
|
${nodeB}.succeed("iptables -D INPUT -s " + node_a + " -j DROP")
|
|
${nodeA}.systemctl("restart nebula@smoke.service")
|
|
${nodeB}.systemctl("restart nebula@smoke.service")
|
|
'';
|
|
in ''
|
|
# Create the certificate and sign the lighthouse's keys.
|
|
${setUpPrivateKey "lighthouse"}
|
|
lighthouse.succeed(
|
|
"mkdir -p /etc/nebula",
|
|
'nebula-cert ca -name "Smoke Test" -out-crt /etc/nebula/ca.crt -out-key /etc/nebula/ca.key',
|
|
'nebula-cert sign -ca-crt /etc/nebula/ca.crt -ca-key /etc/nebula/ca.key -name "lighthouse" -groups "lighthouse" -ip "10.0.100.1/24" -out-crt /etc/nebula/lighthouse.crt -out-key /etc/nebula/lighthouse.key',
|
|
'chown -R nebula-smoke:nebula-smoke /etc/nebula'
|
|
)
|
|
|
|
# Reboot the lighthouse and verify that the nebula service comes up on boot.
|
|
# Since rebooting takes a while, we'll just restart the service on the other nodes.
|
|
lighthouse.shutdown()
|
|
lighthouse.start()
|
|
lighthouse.wait_for_unit("nebula@smoke.service")
|
|
lighthouse.succeed("ping -c5 10.0.100.1")
|
|
|
|
# Create keys for allowAny's nebula service and test that it comes up.
|
|
${setUpPrivateKey "allowAny"}
|
|
${signKeysFor "allowAny" "10.0.100.2/24"}
|
|
${restartAndCheckNebula "allowAny" "10.0.100.2"}
|
|
|
|
# Create keys for allowFromLighthouse's nebula service and test that it comes up.
|
|
${setUpPrivateKey "allowFromLighthouse"}
|
|
${signKeysFor "allowFromLighthouse" "10.0.100.3/24"}
|
|
${restartAndCheckNebula "allowFromLighthouse" "10.0.100.3"}
|
|
|
|
# Create keys for allowToLighthouse's nebula service and test that it comes up.
|
|
${setUpPrivateKey "allowToLighthouse"}
|
|
${signKeysFor "allowToLighthouse" "10.0.100.4/24"}
|
|
${restartAndCheckNebula "allowToLighthouse" "10.0.100.4"}
|
|
|
|
# Create keys for disabled's nebula service and test that it does not come up.
|
|
${setUpPrivateKey "disabled"}
|
|
${signKeysFor "disabled" "10.0.100.5/24"}
|
|
disabled.fail("systemctl status nebula@smoke.service")
|
|
disabled.fail("ping -c5 10.0.100.5")
|
|
|
|
# The lighthouse can ping allowAny and allowFromLighthouse but not disabled
|
|
lighthouse.succeed("ping -c3 10.0.100.2")
|
|
lighthouse.succeed("ping -c3 10.0.100.3")
|
|
lighthouse.fail("ping -c3 10.0.100.5")
|
|
|
|
# allowAny can ping the lighthouse, but not allowFromLighthouse because of its inbound firewall
|
|
allowAny.succeed("ping -c3 10.0.100.1")
|
|
allowAny.fail("ping -c3 10.0.100.3")
|
|
|
|
# allowFromLighthouse can ping the lighthouse and allowAny
|
|
allowFromLighthouse.succeed("ping -c3 10.0.100.1")
|
|
allowFromLighthouse.succeed("ping -c3 10.0.100.2")
|
|
|
|
# block allowFromLighthouse <-> allowAny, and allowFromLighthouse -> allowAny should still work.
|
|
${blockTrafficBetween "allowFromLighthouse" "allowAny"}
|
|
allowFromLighthouse.succeed("ping -c10 10.0.100.2")
|
|
${allowTrafficBetween "allowFromLighthouse" "allowAny"}
|
|
allowFromLighthouse.succeed("ping -c10 10.0.100.2")
|
|
|
|
# allowToLighthouse can ping the lighthouse but not allowAny or allowFromLighthouse
|
|
allowToLighthouse.succeed("ping -c3 10.0.100.1")
|
|
allowToLighthouse.fail("ping -c3 10.0.100.2")
|
|
allowToLighthouse.fail("ping -c3 10.0.100.3")
|
|
|
|
# allowAny can ping allowFromLighthouse now that allowFromLighthouse pinged it first
|
|
allowAny.succeed("ping -c3 10.0.100.3")
|
|
|
|
# block allowAny <-> allowFromLighthouse, and allowAny -> allowFromLighthouse should still work.
|
|
${blockTrafficBetween "allowAny" "allowFromLighthouse"}
|
|
allowFromLighthouse.succeed("ping -c10 10.0.100.2")
|
|
allowAny.succeed("ping -c10 10.0.100.3")
|
|
${allowTrafficBetween "allowAny" "allowFromLighthouse"}
|
|
allowFromLighthouse.succeed("ping -c10 10.0.100.2")
|
|
allowAny.succeed("ping -c10 10.0.100.3")
|
|
|
|
# allowToLighthouse can ping allowAny if allowAny pings it first
|
|
allowAny.succeed("ping -c3 10.0.100.4")
|
|
allowToLighthouse.succeed("ping -c3 10.0.100.2")
|
|
|
|
# block allowToLighthouse <-> allowAny, and allowAny <-> allowToLighthouse should still work.
|
|
${blockTrafficBetween "allowAny" "allowToLighthouse"}
|
|
allowAny.succeed("ping -c10 10.0.100.4")
|
|
allowToLighthouse.succeed("ping -c10 10.0.100.2")
|
|
${allowTrafficBetween "allowAny" "allowToLighthouse"}
|
|
allowAny.succeed("ping -c10 10.0.100.4")
|
|
allowToLighthouse.succeed("ping -c10 10.0.100.2")
|
|
|
|
# block lighthouse <-> allowFromLighthouse and allowAny <-> allowFromLighthouse; allowFromLighthouse won't get to allowAny
|
|
${blockTrafficBetween "allowFromLighthouse" "lighthouse"}
|
|
${blockTrafficBetween "allowFromLighthouse" "allowAny"}
|
|
allowFromLighthouse.fail("ping -c3 10.0.100.2")
|
|
${allowTrafficBetween "allowFromLighthouse" "lighthouse"}
|
|
${allowTrafficBetween "allowFromLighthouse" "allowAny"}
|
|
allowFromLighthouse.succeed("ping -c3 10.0.100.2")
|
|
|
|
# block lighthouse <-> allowAny, allowAny <-> allowFromLighthouse, and allowAny <-> allowToLighthouse; it won't get to allowFromLighthouse or allowToLighthouse
|
|
${blockTrafficBetween "allowAny" "lighthouse"}
|
|
${blockTrafficBetween "allowAny" "allowFromLighthouse"}
|
|
${blockTrafficBetween "allowAny" "allowToLighthouse"}
|
|
allowFromLighthouse.fail("ping -c3 10.0.100.2")
|
|
allowAny.fail("ping -c3 10.0.100.3")
|
|
allowAny.fail("ping -c3 10.0.100.4")
|
|
${allowTrafficBetween "allowAny" "lighthouse"}
|
|
${allowTrafficBetween "allowAny" "allowFromLighthouse"}
|
|
${allowTrafficBetween "allowAny" "allowToLighthouse"}
|
|
allowFromLighthouse.succeed("ping -c3 10.0.100.2")
|
|
allowAny.succeed("ping -c3 10.0.100.3")
|
|
allowAny.succeed("ping -c3 10.0.100.4")
|
|
|
|
# block lighthouse <-> allowToLighthouse and allowToLighthouse <-> allowAny; it won't get to allowAny
|
|
${blockTrafficBetween "allowToLighthouse" "lighthouse"}
|
|
${blockTrafficBetween "allowToLighthouse" "allowAny"}
|
|
allowAny.fail("ping -c3 10.0.100.4")
|
|
allowToLighthouse.fail("ping -c3 10.0.100.2")
|
|
${allowTrafficBetween "allowToLighthouse" "lighthouse"}
|
|
${allowTrafficBetween "allowToLighthouse" "allowAny"}
|
|
allowAny.succeed("ping -c3 10.0.100.4")
|
|
allowToLighthouse.succeed("ping -c3 10.0.100.2")
|
|
'';
|
|
})
|