diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
index 2e53929f061f..fb29facd8b5f 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml
@@ -401,6 +401,12 @@
due to upstream dropping support.
+
+
+ k3s supports clusterInit
+ option, and it is enabled by default, for servers.
+
+
diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md
index 1f1d2f34074b..8c543098762b 100644
--- a/nixos/doc/manual/release-notes/rl-2211.section.md
+++ b/nixos/doc/manual/release-notes/rl-2211.section.md
@@ -141,6 +141,8 @@ Use `configure.packages` instead.
- `k3s` no longer supports docker as runtime due to upstream dropping support.
+- `k3s` supports `clusterInit` option, and it is enabled by default, for servers.
+
## Other Notable Changes {#sec-release-22.11-notable-changes}
diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix
index a1f6d4ecdfad..d0269fde8936 100644
--- a/nixos/modules/services/cluster/k3s/default.nix
+++ b/nixos/modules/services/cluster/k3s/default.nix
@@ -25,7 +25,17 @@ in
role = mkOption {
description = lib.mdDoc ''
Whether k3s should run as a server or agent.
- Note that the server, by default, also runs as an agent.
+
+ If it's a server:
+
+ - By default it also runs workloads as an agent.
+ - Starts by default as a standalone server using an embedded sqlite datastore.
+ - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
+ - Configure `serverAddr` to join an already-initialized HA cluster.
+
+ If it's an agent:
+
+ - `serverAddr` is required.
'';
default = "server";
type = types.enum [ "server" "agent" ];
@@ -33,15 +43,44 @@ in
serverAddr = mkOption {
type = types.str;
- description = lib.mdDoc "The k3s server to connect to. This option only makes sense for an agent.";
+ description = lib.mdDoc ''
+ The k3s server to connect to.
+
+ Servers and agents need to communicate each other. Read
+ [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
+ to know how to configure the firewall.
+ '';
example = "https://10.0.0.10:6443";
default = "";
};
+ clusterInit = mkOption {
+ type = types.bool;
+ default = false;
+ description = lib.mdDoc ''
+ Initialize HA cluster using an embedded etcd datastore.
+
+ If this option is `false` and `role` is `server`
+
+ On a server that was using the default embedded sqlite backend,
+ enabling this option will migrate to an embedded etcd DB.
+
+ If an HA cluster using the embedded etcd datastore was already initialized,
+ this option has no effect.
+
+ This option only makes sense in a server that is not connecting to another server.
+
+ If you are configuring an HA cluster with an embedded etcd,
+ the 1st server must have `clusterInit = true`
+ and other servers must connect to it using `serverAddr`.
+ '';
+ };
+
token = mkOption {
type = types.str;
description = lib.mdDoc ''
- The k3s token to use when connecting to the server. This option only makes sense for an agent.
+ The k3s token to use when connecting to a server.
+
WARNING: This option will expose store your token unencrypted world-readable in the nix store.
If this is undesired use the tokenFile option instead.
'';
@@ -50,7 +89,7 @@ in
tokenFile = mkOption {
type = types.nullOr types.path;
- description = lib.mdDoc "File path containing k3s token to use when connecting to the server. This option only makes sense for an agent.";
+ description = lib.mdDoc "File path containing k3s token to use when connecting to the server.";
default = null;
};
@@ -86,6 +125,14 @@ in
assertion = cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != "";
message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
}
+ {
+ assertion = cfg.role == "agent" -> !cfg.disableAgent;
+ message = "disableAgent must be false if role is 'agent'";
+ }
+ {
+ assertion = cfg.role == "agent" -> !cfg.clusterInit;
+ message = "clusterInit must be false if role is 'agent'";
+ }
];
environment.systemPackages = [ config.services.k3s.package ];
@@ -111,6 +158,7 @@ in
[
"${cfg.package}/bin/k3s ${cfg.role}"
]
+ ++ (optional cfg.clusterInit "--cluster-init")
++ (optional cfg.disableAgent "--disable-agent")
++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
++ (optional (cfg.token != "") "--token ${cfg.token}")
diff --git a/nixos/tests/k3s/multi-node.nix b/nixos/tests/k3s/multi-node.nix
index afb8c78f2339..ae9609fbccc9 100644
--- a/nixos/tests/k3s/multi-node.nix
+++ b/nixos/tests/k3s/multi-node.nix
@@ -53,9 +53,10 @@ import ../make-test-python.nix ({ pkgs, ... }:
enable = true;
role = "server";
package = pkgs.k3s;
+ clusterInit = true;
extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local --node-ip 192.168.1.1";
};
- networking.firewall.allowedTCPPorts = [ 6443 ];
+ networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
networking.firewall.allowedUDPPorts = [ 8472 ];
networking.firewall.trustedInterfaces = [ "flannel.1" ];
networking.useDHCP = false;
@@ -65,6 +66,28 @@ import ../make-test-python.nix ({ pkgs, ... }:
];
};
+ server2 = { pkgs, ... }: {
+ environment.systemPackages = with pkgs; [ gzip jq ];
+ virtualisation.memorySize = 1536;
+ virtualisation.diskSize = 4096;
+
+ services.k3s = {
+ inherit tokenFile;
+ enable = true;
+ serverAddr = "https://192.168.1.1:6443";
+ clusterInit = false;
+ extraFlags = "--no-deploy coredns,servicelb,traefik,local-storage,metrics-server --pause-image test.local/pause:local --node-ip 192.168.1.3";
+ };
+ networking.firewall.allowedTCPPorts = [ 2379 2380 6443 ];
+ networking.firewall.allowedUDPPorts = [ 8472 ];
+ networking.firewall.trustedInterfaces = [ "flannel.1" ];
+ networking.useDHCP = false;
+ networking.defaultGateway = "192.168.1.3";
+ networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
+ { address = "192.168.1.3"; prefixLength = 24; }
+ ];
+ };
+
agent = { pkgs, ... }: {
virtualisation.memorySize = 1024;
virtualisation.diskSize = 2048;
@@ -72,7 +95,7 @@ import ../make-test-python.nix ({ pkgs, ... }:
inherit tokenFile;
enable = true;
role = "agent";
- serverAddr = "https://192.168.1.1:6443";
+ serverAddr = "https://192.168.1.3:6443";
extraFlags = "--pause-image test.local/pause:local --node-ip 192.168.1.2";
};
networking.firewall.allowedTCPPorts = [ 6443 ];
@@ -91,9 +114,9 @@ import ../make-test-python.nix ({ pkgs, ... }:
};
testScript = ''
- start_all()
- machines = [server, agent]
+ machines = [server, server2, agent]
for m in machines:
+ m.start()
m.wait_for_unit("k3s")
# wait for the agent to show up