503 lines
17 KiB
Nix
503 lines
17 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
|
|
with lib;
|
|
let
|
|
cfg = config.services.k3s;
|
|
removeOption =
|
|
config: instruction:
|
|
lib.mkRemovedOptionModule (
|
|
[
|
|
"services"
|
|
"k3s"
|
|
]
|
|
++ config
|
|
) instruction;
|
|
|
|
manifestDir = "/var/lib/rancher/k3s/server/manifests";
|
|
chartDir = "/var/lib/rancher/k3s/server/static/charts";
|
|
imageDir = "/var/lib/rancher/k3s/agent/images";
|
|
|
|
manifestModule =
|
|
let
|
|
mkTarget =
|
|
name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
|
|
in
|
|
lib.types.submodule (
|
|
{
|
|
name,
|
|
config,
|
|
options,
|
|
...
|
|
}:
|
|
{
|
|
options = {
|
|
enable = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = true;
|
|
description = "Whether this manifest file should be generated.";
|
|
};
|
|
|
|
target = lib.mkOption {
|
|
type = lib.types.nonEmptyStr;
|
|
example = lib.literalExpression "manifest.yaml";
|
|
description = ''
|
|
Name of the symlink (relative to {file}`${manifestDir}`).
|
|
Defaults to the attribute name.
|
|
'';
|
|
};
|
|
|
|
content = lib.mkOption {
|
|
type = with lib.types; nullOr (either attrs (listOf attrs));
|
|
default = null;
|
|
description = ''
|
|
Content of the manifest file. A single attribute set will
|
|
generate a single document YAML file. A list of attribute sets
|
|
will generate multiple documents separated by `---` in a single
|
|
YAML file.
|
|
'';
|
|
};
|
|
|
|
source = lib.mkOption {
|
|
type = lib.types.path;
|
|
example = lib.literalExpression "./manifests/app.yaml";
|
|
description = ''
|
|
Path of the source `.yaml` file.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = {
|
|
target = lib.mkDefault (mkTarget name);
|
|
source = lib.mkIf (config.content != null) (
|
|
let
|
|
name' = "k3s-manifest-" + builtins.baseNameOf name;
|
|
docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
|
|
yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
|
|
mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
|
|
mkSource =
|
|
value:
|
|
if builtins.isList value then
|
|
pkgs.concatText name' (
|
|
lib.concatMap (x: [
|
|
yamlDocSeparator
|
|
(mkYaml docName x)
|
|
]) value
|
|
)
|
|
else
|
|
mkYaml name' value;
|
|
in
|
|
lib.mkDerivedConfig options.content mkSource
|
|
);
|
|
};
|
|
}
|
|
);
|
|
|
|
enabledManifests = with builtins; filter (m: m.enable) (attrValues cfg.manifests);
|
|
linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
|
|
linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
|
|
linkChartEntry =
|
|
let
|
|
mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
|
|
in
|
|
name: value:
|
|
"${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
|
|
|
|
activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
|
|
${lib.optionalString (
|
|
builtins.length enabledManifests > 0
|
|
) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
|
|
${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
|
|
${lib.optionalString (
|
|
builtins.length cfg.images > 0
|
|
) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
|
|
|
|
${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
|
|
${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
|
|
${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
|
|
'';
|
|
in
|
|
{
|
|
imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
|
|
|
|
# interface
|
|
options.services.k3s = {
|
|
enable = mkEnableOption "k3s";
|
|
|
|
package = mkPackageOption pkgs "k3s" { };
|
|
|
|
role = mkOption {
|
|
description = ''
|
|
Whether k3s should run as a server or agent.
|
|
|
|
If it's a server:
|
|
|
|
- By default it also runs workloads as an agent.
|
|
- Starts by default as a standalone server using an embedded sqlite datastore.
|
|
- Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
|
|
- Configure `serverAddr` to join an already-initialized HA cluster.
|
|
|
|
If it's an agent:
|
|
|
|
- `serverAddr` is required.
|
|
'';
|
|
default = "server";
|
|
type = types.enum [
|
|
"server"
|
|
"agent"
|
|
];
|
|
};
|
|
|
|
serverAddr = mkOption {
|
|
type = types.str;
|
|
description = ''
|
|
The k3s server to connect to.
|
|
|
|
Servers and agents need to communicate each other. Read
|
|
[the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
|
|
to know how to configure the firewall.
|
|
'';
|
|
example = "https://10.0.0.10:6443";
|
|
default = "";
|
|
};
|
|
|
|
clusterInit = mkOption {
|
|
type = types.bool;
|
|
default = false;
|
|
description = ''
|
|
Initialize HA cluster using an embedded etcd datastore.
|
|
|
|
If this option is `false` and `role` is `server`
|
|
|
|
On a server that was using the default embedded sqlite backend,
|
|
enabling this option will migrate to an embedded etcd DB.
|
|
|
|
If an HA cluster using the embedded etcd datastore was already initialized,
|
|
this option has no effect.
|
|
|
|
This option only makes sense in a server that is not connecting to another server.
|
|
|
|
If you are configuring an HA cluster with an embedded etcd,
|
|
the 1st server must have `clusterInit = true`
|
|
and other servers must connect to it using `serverAddr`.
|
|
'';
|
|
};
|
|
|
|
token = mkOption {
|
|
type = types.str;
|
|
description = ''
|
|
The k3s token to use when connecting to a server.
|
|
|
|
WARNING: This option will expose store your token unencrypted world-readable in the nix store.
|
|
If this is undesired use the tokenFile option instead.
|
|
'';
|
|
default = "";
|
|
};
|
|
|
|
tokenFile = mkOption {
|
|
type = types.nullOr types.path;
|
|
description = "File path containing k3s token to use when connecting to the server.";
|
|
default = null;
|
|
};
|
|
|
|
extraFlags = mkOption {
|
|
description = "Extra flags to pass to the k3s command.";
|
|
type = with types; either str (listOf str);
|
|
default = [ ];
|
|
example = [
|
|
"--no-deploy traefik"
|
|
"--cluster-cidr 10.24.0.0/16"
|
|
];
|
|
};
|
|
|
|
disableAgent = mkOption {
|
|
type = types.bool;
|
|
default = false;
|
|
description = "Only run the server. This option only makes sense for a server.";
|
|
};
|
|
|
|
environmentFile = mkOption {
|
|
type = types.nullOr types.path;
|
|
description = ''
|
|
File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See systemd.exec(5).
|
|
'';
|
|
default = null;
|
|
};
|
|
|
|
configPath = mkOption {
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
|
|
};
|
|
|
|
manifests = mkOption {
|
|
type = types.attrsOf manifestModule;
|
|
default = { };
|
|
example = lib.literalExpression ''
|
|
deployment.source = ../manifests/deployment.yaml;
|
|
my-service = {
|
|
enable = false;
|
|
target = "app-service.yaml";
|
|
content = {
|
|
apiVersion = "v1";
|
|
kind = "Service";
|
|
metadata = {
|
|
name = "app-service";
|
|
};
|
|
spec = {
|
|
selector = {
|
|
"app.kubernetes.io/name" = "MyApp";
|
|
};
|
|
ports = [
|
|
{
|
|
name = "name-of-service-port";
|
|
protocol = "TCP";
|
|
port = 80;
|
|
targetPort = "http-web-svc";
|
|
}
|
|
];
|
|
};
|
|
}
|
|
};
|
|
|
|
nginx.content = [
|
|
{
|
|
apiVersion = "v1";
|
|
kind = "Pod";
|
|
metadata = {
|
|
name = "nginx";
|
|
labels = {
|
|
"app.kubernetes.io/name" = "MyApp";
|
|
};
|
|
};
|
|
spec = {
|
|
containers = [
|
|
{
|
|
name = "nginx";
|
|
image = "nginx:1.14.2";
|
|
ports = [
|
|
{
|
|
containerPort = 80;
|
|
name = "http-web-svc";
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
}
|
|
{
|
|
apiVersion = "v1";
|
|
kind = "Service";
|
|
metadata = {
|
|
name = "nginx-service";
|
|
};
|
|
spec = {
|
|
selector = {
|
|
"app.kubernetes.io/name" = "MyApp";
|
|
};
|
|
ports = [
|
|
{
|
|
name = "name-of-service-port";
|
|
protocol = "TCP";
|
|
port = 80;
|
|
targetPort = "http-web-svc";
|
|
}
|
|
];
|
|
};
|
|
}
|
|
];
|
|
'';
|
|
description = ''
|
|
Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
|
|
Note that deleting manifest files will not remove or otherwise modify the resources
|
|
it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
|
|
as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
|
|
This option only makes sense on server nodes (`role = server`).
|
|
Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
|
|
for further information.
|
|
'';
|
|
};
|
|
|
|
charts = mkOption {
|
|
type = with types; attrsOf (either path package);
|
|
default = { };
|
|
example = lib.literalExpression ''
|
|
nginx = ../charts/my-nginx-chart.tgz;
|
|
redis = ../charts/my-redis-chart.tgz;
|
|
'';
|
|
description = ''
|
|
Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
|
|
The attribute name will be used as the link target (relative to {file}`${chartDir}`).
|
|
The specified charts will only be placed on the file system and made available to the
|
|
Kubernetes APIServer from within the cluster, you may use the
|
|
[k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
|
|
to deploy the charts. This option only makes sense on server nodes
|
|
(`role = server`).
|
|
'';
|
|
};
|
|
|
|
images = mkOption {
|
|
type = with types; listOf package;
|
|
default = [ ];
|
|
example = lib.literalExpression ''
|
|
[
|
|
(pkgs.dockerTools.pullImage {
|
|
imageName = "docker.io/bitnami/keycloak";
|
|
imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
|
|
sha256 = "0imblp0kw9vkcr7sp962jmj20fpmb3hvd3hmf4cs4x04klnq3k90";
|
|
finalImageTag = "21.1.2-debian-11-r0";
|
|
})
|
|
]
|
|
'';
|
|
description = ''
|
|
List of derivations that provide container images.
|
|
All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
|
|
by the k3s agent. This option only makes sense on nodes with an enabled agent.
|
|
'';
|
|
};
|
|
|
|
gracefulNodeShutdown = {
|
|
enable = lib.mkEnableOption ''
|
|
graceful node shutdowns where the kubelet attempts to detect
|
|
node system shutdown and terminates pods running on the node. See the
|
|
[documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
|
|
for further information.
|
|
'';
|
|
|
|
shutdownGracePeriod = lib.mkOption {
|
|
type = lib.types.nonEmptyStr;
|
|
default = "30s";
|
|
example = "1m30s";
|
|
description = ''
|
|
Specifies the total duration that the node should delay the shutdown by. This is the total
|
|
grace period for pod termination for both regular and critical pods.
|
|
'';
|
|
};
|
|
|
|
shutdownGracePeriodCriticalPods = lib.mkOption {
|
|
type = lib.types.nonEmptyStr;
|
|
default = "10s";
|
|
example = "15s";
|
|
description = ''
|
|
Specifies the duration used to terminate critical pods during a node shutdown. This should be
|
|
less than `shutdownGracePeriod`.
|
|
'';
|
|
};
|
|
};
|
|
|
|
extraKubeletConfig = lib.mkOption {
|
|
type = with lib.types; attrsOf anything;
|
|
default = { };
|
|
example = {
|
|
podsPerCore = 3;
|
|
memoryThrottlingFactor = 0.69;
|
|
containerLogMaxSize = "5Mi";
|
|
};
|
|
description = ''
|
|
Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
|
|
configuration that can be configured via a file is defined by the
|
|
[KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
|
|
struct. See the
|
|
[documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
|
|
for further information.
|
|
'';
|
|
};
|
|
};
|
|
|
|
# implementation
|
|
|
|
config = mkIf cfg.enable {
|
|
warnings =
|
|
(lib.optional (cfg.role != "server" && cfg.manifests != { })
|
|
"k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
|
|
)
|
|
++ (lib.optional (cfg.role != "server" && cfg.charts != { })
|
|
"k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
|
|
)
|
|
++ (lib.optional (cfg.disableAgent && cfg.images != [ ])
|
|
"k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node"
|
|
);
|
|
|
|
assertions = [
|
|
{
|
|
assertion = cfg.role == "agent" -> (cfg.configPath != null || cfg.serverAddr != "");
|
|
message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
|
|
}
|
|
{
|
|
assertion =
|
|
cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != "";
|
|
message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
|
|
}
|
|
{
|
|
assertion = cfg.role == "agent" -> !cfg.disableAgent;
|
|
message = "disableAgent must be false if role is 'agent'";
|
|
}
|
|
{
|
|
assertion = cfg.role == "agent" -> !cfg.clusterInit;
|
|
message = "clusterInit must be false if role is 'agent'";
|
|
}
|
|
];
|
|
|
|
environment.systemPackages = [ config.services.k3s.package ];
|
|
|
|
systemd.services.k3s =
|
|
let
|
|
kubeletParams =
|
|
(lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
|
|
inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
|
|
})
|
|
// cfg.extraKubeletConfig;
|
|
kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
|
|
{
|
|
apiVersion = "kubelet.config.k8s.io/v1beta1";
|
|
kind = "KubeletConfiguration";
|
|
}
|
|
// kubeletParams
|
|
);
|
|
in
|
|
{
|
|
description = "k3s service";
|
|
after = [
|
|
"firewall.service"
|
|
"network-online.target"
|
|
];
|
|
wants = [
|
|
"firewall.service"
|
|
"network-online.target"
|
|
];
|
|
wantedBy = [ "multi-user.target" ];
|
|
path = optional config.boot.zfs.enabled config.boot.zfs.package;
|
|
serviceConfig = {
|
|
# See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
|
|
Type = if cfg.role == "agent" then "exec" else "notify";
|
|
KillMode = "process";
|
|
Delegate = "yes";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
LimitNOFILE = 1048576;
|
|
LimitNPROC = "infinity";
|
|
LimitCORE = "infinity";
|
|
TasksMax = "infinity";
|
|
EnvironmentFile = cfg.environmentFile;
|
|
ExecStartPre = activateK3sContent;
|
|
ExecStart = concatStringsSep " \\\n " (
|
|
[ "${cfg.package}/bin/k3s ${cfg.role}" ]
|
|
++ (optional cfg.clusterInit "--cluster-init")
|
|
++ (optional cfg.disableAgent "--disable-agent")
|
|
++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
|
|
++ (optional (cfg.token != "") "--token ${cfg.token}")
|
|
++ (optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
|
|
++ (optional (cfg.configPath != null) "--config ${cfg.configPath}")
|
|
++ (optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
|
|
++ (lib.flatten cfg.extraFlags)
|
|
);
|
|
};
|
|
};
|
|
};
|
|
|
|
meta.maintainers = lib.teams.k3s.members;
|
|
}
|