115 lines
4 KiB
Nix
115 lines
4 KiB
Nix
{ system ? builtins.currentSystem,
|
|
config ? {},
|
|
pkgs ? import ../../.. { inherit system config; }
|
|
}:
|
|
|
|
with import ../../lib/testing-python.nix { inherit system pkgs; };
|
|
with pkgs.lib;
|
|
|
|
let
|
|
mkKubernetesBaseTest =
|
|
{ name, domain ? "my.zyx", test, machines
|
|
, extraConfiguration ? null }:
|
|
let
|
|
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
|
|
master = machines.${masterName};
|
|
extraHosts = ''
|
|
${master.ip} etcd.${domain}
|
|
${master.ip} api.${domain}
|
|
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
|
|
'';
|
|
kubectl = with pkgs; runCommand "wrap-kubectl" { buildInputs = [ makeWrapper ]; } ''
|
|
mkdir -p $out/bin
|
|
makeWrapper ${pkgs.kubernetes}/bin/kubectl $out/bin/kubectl --set KUBECONFIG "/etc/kubernetes/cluster-admin.kubeconfig"
|
|
'';
|
|
in makeTest {
|
|
inherit name;
|
|
|
|
nodes = mapAttrs (machineName: machine:
|
|
{ config, pkgs, lib, nodes, ... }:
|
|
mkMerge [
|
|
{
|
|
boot.postBootCommands = "rm -fr /var/lib/kubernetes/secrets /tmp/shared/*";
|
|
virtualisation.memorySize = mkDefault 1536;
|
|
virtualisation.diskSize = mkDefault 4096;
|
|
networking = {
|
|
inherit domain extraHosts;
|
|
primaryIPAddress = mkForce machine.ip;
|
|
|
|
firewall = {
|
|
allowedTCPPorts = [
|
|
10250 # kubelet
|
|
];
|
|
trustedInterfaces = ["mynet"];
|
|
|
|
extraCommands = concatMapStrings (node: ''
|
|
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
|
|
'') (attrValues nodes);
|
|
};
|
|
};
|
|
programs.bash.enableCompletion = true;
|
|
environment.systemPackages = [ kubectl ];
|
|
services.flannel.iface = "eth1";
|
|
services.kubernetes = {
|
|
addons.dashboard.enable = true;
|
|
proxy.hostname = "${masterName}.${domain}";
|
|
|
|
easyCerts = true;
|
|
inherit (machine) roles;
|
|
apiserver = {
|
|
securePort = 443;
|
|
advertiseAddress = master.ip;
|
|
};
|
|
masterAddress = "${masterName}.${config.networking.domain}";
|
|
# workaround for:
|
|
# https://github.com/kubernetes/kubernetes/issues/102676
|
|
# (workaround from) https://github.com/kubernetes/kubernetes/issues/95488
|
|
kubelet.extraOpts = ''\
|
|
--cgroups-per-qos=false \
|
|
--enforce-node-allocatable="" \
|
|
'';
|
|
};
|
|
}
|
|
(optionalAttrs (any (role: role == "master") machine.roles) {
|
|
networking.firewall.allowedTCPPorts = [
|
|
443 # kubernetes apiserver
|
|
];
|
|
})
|
|
(optionalAttrs (machine ? extraConfiguration) (machine.extraConfiguration { inherit config pkgs lib nodes; }))
|
|
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
|
|
]
|
|
) machines;
|
|
|
|
testScript = ''
|
|
start_all()
|
|
'' + test;
|
|
};
|
|
|
|
mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
|
|
machines = {
|
|
machine1 = {
|
|
roles = ["master"];
|
|
ip = "192.168.1.1";
|
|
};
|
|
machine2 = {
|
|
roles = ["node"];
|
|
ip = "192.168.1.2";
|
|
};
|
|
};
|
|
} // attrs // {
|
|
name = "kubernetes-${attrs.name}-multinode";
|
|
});
|
|
|
|
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
|
|
machines = {
|
|
machine1 = {
|
|
roles = ["master" "node"];
|
|
ip = "192.168.1.1";
|
|
};
|
|
};
|
|
} // attrs // {
|
|
name = "kubernetes-${attrs.name}-singlenode";
|
|
});
|
|
in {
|
|
inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
|
|
}
|