112 lines
3.9 KiB
Nix
112 lines
3.9 KiB
Nix
|
{ system ? builtins.currentSystem,
|
||
|
|
config ? {},
|
||
|
|
pkgs ? import ../../.. { inherit system config; }
|
||
|
|
}:
|
||
|
|
|
||
|
|
with import ../../lib/testing-python.nix { inherit system pkgs; };
|
||
|
|
with pkgs.lib;
|
||
|
|
|
||
|
|
let
|
||
|
|
mkKubernetesBaseTest =
|
||
|
|
{ name, domain ? "my.zyx", test, machines
|
||
|
|
, extraConfiguration ? null }:
|
||
|
|
let
|
||
|
|
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
|
||
|
|
master = machines.${masterName};
|
||
|
|
extraHosts = ''
|
||
|
|
${master.ip} etcd.${domain}
|
||
|
|
${master.ip} api.${domain}
|
||
|
|
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
|
||
|
|
'';
|
||
|
|
wrapKubectl = with pkgs; runCommand "wrap-kubectl" { nativeBuildInputs = [ makeWrapper ]; } ''
|
||
|
|
mkdir -p $out/bin
|
||
|
|
makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
|
||
|
|
'';
|
||
|
|
in makeTest {
|
||
|
|
inherit name;
|
||
|
|
|
||
|
|
nodes = mapAttrs (machineName: machine:
|
||
|
|
{ config, pkgs, lib, nodes, ... }:
|
||
|
|
mkMerge [
|
||
|
|
{
|
||
|
|
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
||
|
|
virtualisation.memorySize = mkDefault 1536;
|
||
|
|
virtualisation.diskSize = mkDefault 4096;
|
||
|
|
networking = {
|
||
|
|
inherit domain extraHosts;
|
||
|
|
primaryIPAddress = mkForce machine.ip;
|
||
|
|
|
||
|
|
firewall = {
|
||
|
|
allowedTCPPorts = [
|
||
|
|
10250 # kubelet
|
||
|
|
];
|
||
|
|
trustedInterfaces = ["mynet"];
|
||
|
|
|
||
|
|
extraCommands = concatMapStrings (node: ''
|
||
|
|
iptables -A INPUT -s ${node.networking.primaryIPAddress} -j ACCEPT
|
||
|
|
'') (attrValues nodes);
|
||
|
|
};
|
||
|
|
};
|
||
|
|
programs.bash.completion.enable = true;
|
||
|
|
environment.systemPackages = [ wrapKubectl ];
|
||
|
|
services.flannel.iface = "eth1";
|
||
|
|
services.kubernetes = {
|
||
|
|
proxy.hostname = "${masterName}.${domain}";
|
||
|
|
|
||
|
|
easyCerts = true;
|
||
|
|
inherit (machine) roles;
|
||
|
|
apiserver = {
|
||
|
|
securePort = 443;
|
||
|
|
advertiseAddress = master.ip;
|
||
|
|
};
|
||
|
|
# NOTE: what featureGates are useful for testing might change in
|
||
|
|
# the future, see link below to find new ones
|
||
|
|
# https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
|
||
|
|
featureGates = {AnonymousAuthConfigurableEndpoints = true; ConsistentListFromCache = false;};
|
||
|
|
masterAddress = "${masterName}.${config.networking.domain}";
|
||
|
|
};
|
||
|
|
}
|
||
|
|
(optionalAttrs (any (role: role == "master") machine.roles) {
|
||
|
|
networking.firewall.allowedTCPPorts = [
|
||
|
|
443 # kubernetes apiserver
|
||
|
|
];
|
||
|
|
})
|
||
|
|
(optionalAttrs (machine ? extraConfiguration) (machine.extraConfiguration { inherit config pkgs lib nodes; }))
|
||
|
|
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
|
||
|
|
]
|
||
|
|
) machines;
|
||
|
|
|
||
|
|
testScript = ''
|
||
|
|
start_all()
|
||
|
|
'' + test;
|
||
|
|
};
|
||
|
|
|
||
|
|
mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
|
||
|
|
machines = {
|
||
|
|
machine1 = {
|
||
|
|
roles = ["master"];
|
||
|
|
ip = "192.168.1.1";
|
||
|
|
};
|
||
|
|
machine2 = {
|
||
|
|
roles = ["node"];
|
||
|
|
ip = "192.168.1.2";
|
||
|
|
};
|
||
|
|
};
|
||
|
|
} // attrs // {
|
||
|
|
name = "kubernetes-${attrs.name}-multinode";
|
||
|
|
});
|
||
|
|
|
||
|
|
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
|
||
|
|
machines = {
|
||
|
|
machine1 = {
|
||
|
|
roles = ["master" "node"];
|
||
|
|
ip = "192.168.1.1";
|
||
|
|
};
|
||
|
|
};
|
||
|
|
} // attrs // {
|
||
|
|
name = "kubernetes-${attrs.name}-singlenode";
|
||
|
|
});
|
||
|
|
in {
|
||
|
|
inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
|
||
|
|
}
|