dev/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #49516 from johanot/kubedns-to-coredns

Browse Source 
nixos/kubernetes: KubeDNS -> CoreDNS
This commit is contained in:
Sarah Brofeldt 2018-11-06 10:30:49 +01:00 committed by GitHub
commit 81de3e39b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 223 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
nixos/doc/manual/release-notes/rl-1903.xml
View File

@ -224,6 +224,19 @@
supports loading TrueCrypt volumes. supports loading TrueCrypt volumes.
</para> </para>
</listitem> </listitem>
<listitem>
<para>
The Kubernetes DNS addons, kube-dns, has been replaced with CoreDNS.
This change is made in accordance with Kubernetes making CoreDNS the official default
starting from
<link xlink:href="https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#sig-cluster-lifecycle">Kubernetes v1.11</link>.
Please beware that upgrading DNS-addon on existing clusters might induce
minor downtime while the DNS-addon terminates and re-initializes.
Also note that the DNS-service now runs with 2 pod replicas by default.
The desired number of replicas can be configured using:
<option>services.kubernetes.addons.dns.replicas</option>.
</para>
</listitem>
</itemizedlist> </itemizedlist>
</section> </section>
</section> </section>

422
nixos/modules/services/cluster/kubernetes/dns.nix
View File

@ -3,8 +3,13 @@
with lib; with lib;
let let
version = "1.14.10"; version = "1.2.5";
cfg = config.services.kubernetes.addons.dns; cfg = config.services.kubernetes.addons.dns;
ports = {
dns = 10053;
health = 10054;
metrics = 10055;
};
in { in {
options.services.kubernetes.addons.dns = { options.services.kubernetes.addons.dns = {
enable = mkEnableOption "kubernetes dns addon"; enable = mkEnableOption "kubernetes dns addon";
@ -27,49 +32,130 @@ in {
type = types.str; type = types.str;
}; };
kube-dns = mkOption { replicas = mkOption {
description = "Docker image to seed for the kube-dns main container."; description = "Number of DNS pod replicas to deploy in the cluster.";
type = types.attrs; default = 2;
default = { type = types.int;
imageName = "k8s.gcr.io/k8s-dns-kube-dns-amd64";
imageDigest = "sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8";
finalImageTag = version;
sha256 = "0x583znk9smqn0fix7ld8sm5jgaxhqhx3fq97b1wkqm7iwhvl3pj";
};
}; };
dnsmasq-nanny = mkOption { coredns = mkOption {
description = "Docker image to seed for the kube-dns dnsmasq container."; description = "Docker image to seed for the CoreDNS container.";
type = types.attrs; type = types.attrs;
default = { default = {
imageName = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64"; imageName = "coredns/coredns";
imageDigest = "sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8"; imageDigest = "sha256:33c8da20b887ae12433ec5c40bfddefbbfa233d5ce11fb067122e68af30291d6";
finalImageTag = version; finalImageTag = version;
sha256 = "1fihml7s2mfwgac51cbqpylkwbivc8nyhgi4vb820s83zvl8a6y1"; sha256 = "13q19rgwapv27xcs664dw502254yw4zw63insf6g2danidv2mg6i";
};
};
sidecar = mkOption {
description = "Docker image to seed for the kube-dns sidecar container.";
type = types.attrs;
default = {
imageName = "k8s.gcr.io/k8s-dns-sidecar-amd64";
imageDigest = "sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4";
finalImageTag = version;
sha256 = "08l1bv5jgrhvjzpqpbinrkgvv52snc4fzyd8ya9v18ns2klyz7m0";
}; };
}; };
}; };
config = mkIf cfg.enable { config = mkIf cfg.enable {
services.kubernetes.kubelet.seedDockerImages = with pkgs.dockerTools; [ services.kubernetes.kubelet.seedDockerImages =
(pullImage cfg.kube-dns) singleton (pkgs.dockerTools.pullImage cfg.coredns);
(pullImage cfg.dnsmasq-nanny)
(pullImage cfg.sidecar)
];
services.kubernetes.addonManager.addons = { services.kubernetes.addonManager.addons = {
kubedns-deployment = { coredns-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
};
name = "coredns";
namespace = "kube-system";
};
};
coredns-cr = {
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "ClusterRole";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/bootstrapping" = "rbac-defaults";
};
name = "system:coredns";
};
rules = [
{
apiGroups = [ "" ];
resources = [ "endpoints" "services" "pods" "namespaces" ];
verbs = [ "list" "watch" ];
}
{
apiGroups = [ "" ];
resources = [ "nodes" ];
verbs = [ "get" ];
}
];
};
coredns-crb = {
apiVersion = "rbac.authorization.k8s.io/v1beta1";
kind = "ClusterRoleBinding";
metadata = {
annotations = {
"rbac.authorization.kubernetes.io/autoupdate" = "true";
};
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
"kubernetes.io/bootstrapping" = "rbac-defaults";
};
name = "system:coredns";
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "system:coredns";
};
subjects = [
{
kind = "ServiceAccount";
name = "coredns";
namespace = "kube-system";
}
];
};
coredns-cm = {
apiVersion = "v1";
kind = "ConfigMap";
metadata = {
labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true";
};
name = "coredns";
namespace = "kube-system";
};
data = {
Corefile = ".:${toString ports.dns} {
errors
health :${toString ports.health}
kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
prometheus :${toString ports.metrics}
proxy . /etc/resolv.conf
cache 30
loop
reload
loadbalance
}";
};
};
coredns-deploy = {
apiVersion = "extensions/v1beta1"; apiVersion = "extensions/v1beta1";
kind = "Deployment"; kind = "Deployment";
metadata = { metadata = {
@ -77,182 +163,96 @@ in {
"addonmanager.kubernetes.io/mode" = "Reconcile"; "addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns"; "k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true"; "kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "CoreDNS";
}; };
name = "kube-dns"; name = "coredns";
namespace = "kube-system"; namespace = "kube-system";
}; };
spec = { spec = {
selector.matchLabels."k8s-app" = "kube-dns"; replicas = cfg.replicas;
selector = {
matchLabels = { k8s-app = "kube-dns"; };
};
strategy = { strategy = {
rollingUpdate = { rollingUpdate = { maxUnavailable = 1; };
maxSurge = "10%"; type = "RollingUpdate";
maxUnavailable = 0;
};
}; };
template = { template = {
metadata = { metadata = {
annotations."scheduler.alpha.kubernetes.io/critical-pod" = ""; labels = {
labels.k8s-app = "kube-dns"; k8s-app = "kube-dns";
};
}; };
spec = { spec = {
priorityClassName = "system-cluster-critical";
containers = [ containers = [
{ {
name = "kubedns"; args = [ "-conf" "/etc/coredns/Corefile" ];
image = with cfg.kube-dns; "${imageName}:${finalImageTag}"; image = with cfg.coredns; "${imageName}:${finalImageTag}";
imagePullPolicy = "Never";
livenessProbe = {
failureThreshold = 5;
httpGet = {
path = "/health";
port = ports.health;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
name = "coredns";
ports = [
{
containerPort = ports.dns;
name = "dns";
protocol = "UDP";
}
{
containerPort = ports.dns;
name = "dns-tcp";
protocol = "TCP";
}
{
containerPort = ports.metrics;
name = "metrics";
protocol = "TCP";
}
];
resources = { resources = {
limits.memory = "170Mi"; limits = {
memory = "170Mi";
};
requests = { requests = {
cpu = "100m"; cpu = "100m";
memory = "70Mi"; memory = "70Mi";
}; };
}; };
livenessProbe = { securityContext = {
failureThreshold = 5; allowPrivilegeEscalation = false;
httpGet = { capabilities = {
path = "/healthcheck/kubedns"; drop = [ "all" ];
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
successThreshold = 1;
timeoutSeconds = 5;
};
readinessProbe = {
httpGet = {
path = "/readiness";
port = 8081;
scheme = "HTTP";
};
initialDelaySeconds = 3;
timeoutSeconds = 5;
};
args = [
"--domain=${cfg.clusterDomain}"
"--dns-port=10053"
"--config-dir=/kube-dns-config"
"--v=2"
];
env = [
{
name = "PROMETHEUS_PORT";
value = "10055";
}
];
ports = [
{
containerPort = 10053;
name = "dns-local";
protocol = "UDP";
}
{
containerPort = 10053;
name = "dns-tcp-local";
protocol = "TCP";
}
{
containerPort = 10055;
name = "metrics";
protocol = "TCP";
}
];
volumeMounts = [
{
mountPath = "/kube-dns-config";
name = "kube-dns-config";
}
];
}
{
name = "dnsmasq";
image = with cfg.dnsmasq-nanny; "${imageName}:${finalImageTag}";
livenessProbe = {
httpGet = {
path = "/healthcheck/dnsmasq";
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
timeoutSeconds = 5;
successThreshold = 1;
failureThreshold = 5;
};
args = [
"-v=2"
"-logtostderr"
"-configDir=/etc/k8s/dns/dnsmasq-nanny"
"-restartDnsmasq=true"
"--"
"-k"
"--cache-size=1000"
"--log-facility=-"
"--server=/${cfg.clusterDomain}/127.0.0.1#10053"
"--server=/in-addr.arpa/127.0.0.1#10053"
"--server=/ip6.arpa/127.0.0.1#10053"
];
ports = [
{
containerPort = 53;
name = "dns";
protocol = "UDP";
}
{
containerPort = 53;
name = "dns-tcp";
protocol = "TCP";
}
];
resources = {
requests = {
cpu = "150m";
memory = "20Mi";
}; };
readOnlyRootFilesystem = true;
}; };
volumeMounts = [ volumeMounts = [
{ {
mountPath = "/etc/k8s/dns/dnsmasq-nanny"; mountPath = "/etc/coredns";
name = "kube-dns-config"; name = "config-volume";
readOnly = true;
} }
]; ];
} }
{
name = "sidecar";
image = with cfg.sidecar; "${imageName}:${finalImageTag}";
livenessProbe = {
httpGet = {
path = "/metrics";
port = 10054;
scheme = "HTTP";
};
initialDelaySeconds = 60;
timeoutSeconds = 5;
successThreshold = 1;
failureThreshold = 5;
};
args = [
"--v=2"
"--logtostderr"
"--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cfg.clusterDomain},5,A"
"--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cfg.clusterDomain},5,A"
];
ports = [
{
containerPort = 10054;
name = "metrics";
protocol = "TCP";
}
];
resources = {
requests = {
cpu = "10m";
memory = "20Mi";
};
};
}
]; ];
dnsPolicy = "Default"; dnsPolicy = "Default";
serviceAccountName = "kube-dns"; nodeSelector = {
"beta.kubernetes.io/os" = "linux";
};
serviceAccountName = "coredns";
tolerations = [ tolerations = [
{
effect = "NoSchedule";
key = "node-role.kubernetes.io/master";
}
{ {
key = "CriticalAddonsOnly"; key = "CriticalAddonsOnly";
operator = "Exists"; operator = "Exists";
@ -261,10 +261,15 @@ in {
volumes = [ volumes = [
{ {
configMap = { configMap = {
name = "kube-dns"; items = [
optional = true; {
key = "Corefile";
path = "Corefile";
}
];
name = "coredns";
}; };
name = "kube-dns-config"; name = "config-volume";
} }
]; ];
}; };
@ -272,51 +277,40 @@ in {
}; };
}; };
kubedns-svc = { coredns-svc = {
apiVersion = "v1"; apiVersion = "v1";
kind = "Service"; kind = "Service";
metadata = { metadata = {
annotations = {
"prometheus.io/port" = toString ports.metrics;
"prometheus.io/scrape" = "true";
};
labels = { labels = {
"addonmanager.kubernetes.io/mode" = "Reconcile"; "addonmanager.kubernetes.io/mode" = "Reconcile";
"k8s-app" = "kube-dns"; "k8s-app" = "kube-dns";
"kubernetes.io/cluster-service" = "true"; "kubernetes.io/cluster-service" = "true";
"kubernetes.io/name" = "KubeDNS"; "kubernetes.io/name" = "CoreDNS";
}; };
name = "kube-dns"; name = "kube-dns";
namespace = "kube-system"; namespace = "kube-system";
}; };
spec = { spec = {
clusterIP = cfg.clusterIp; clusterIP = cfg.clusterIp;
ports = [ ports = [
{name = "dns"; port = 53; protocol = "UDP";} {
{name = "dns-tcp"; port = 53; protocol = "TCP";} name = "dns";
port = 53;
targetPort = ports.dns;
protocol = "UDP";
}
{
name = "dns-tcp";
port = 53;
targetPort = ports.dns;
protocol = "TCP";
}
]; ];
selector.k8s-app = "kube-dns"; selector = { k8s-app = "kube-dns"; };
};
};
kubedns-sa = {
apiVersion = "v1";
kind = "ServiceAccount";
metadata = {
name = "kube-dns";
namespace = "kube-system";
labels = {
"kubernetes.io/cluster-service" = "true";
"addonmanager.kubernetes.io/mode" = "Reconcile";
};
};
};
kubedns-cm = {
apiVersion = "v1";
kind = "ConfigMap";
metadata = {
name = "kube-dns";
namespace = "kube-system";
labels = {
"addonmanager.kubernetes.io/mode" = "EnsureExists";
};
}; };
}; };
}; };

4
nixos/tests/kubernetes/dns.nix
View File

@ -87,7 +87,7 @@ let
# check if pods are running # check if pods are running
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running"); $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running"); $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'"); $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'");
# check dns on host (dnsmasq) # check dns on host (dnsmasq)
$machine1->succeed("host redis.default.svc.cluster.local"); $machine1->succeed("host redis.default.svc.cluster.local");
@ -111,7 +111,7 @@ let
# check if pods are running # check if pods are running
$machine1->waitUntilSucceeds("kubectl get pod redis | grep Running"); $machine1->waitUntilSucceeds("kubectl get pod redis | grep Running");
$machine1->waitUntilSucceeds("kubectl get pod probe | grep Running"); $machine1->waitUntilSucceeds("kubectl get pod probe | grep Running");
$machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'kube-dns.*3/3'"); $machine1->waitUntilSucceeds("kubectl get pods -n kube-system | grep 'coredns.*1/1'");
# check dns on hosts (dnsmasq) # check dns on hosts (dnsmasq)
$machine1->succeed("host redis.default.svc.cluster.local"); $machine1->succeed("host redis.default.svc.cluster.local");