dev/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

nixos/hadoop: fix incorrect merging of yarnSiteInternal

Browse Source
This commit is contained in:
illustris 2022-05-13 10:55:16 +05:30 committed by pennae
parent ac403b83fb
commit 1285a586c5
3 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nixos/modules/services/cluster/hadoop/hdfs.nix
View File

@ -158,8 +158,8 @@ in
50010 # datanode.address 50010 # datanode.address
50020 # datanode.ipc.address 50020 # datanode.ipc.address
]; ];
extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = let d = cfg.hdfs.datanode.dataDirs; in extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = mkIf (cfg.hdfs.datanode.dataDirs!= null)
if (d!= null) then (concatMapStringsSep "," (x: "["+x.type+"]file://"+x.path) cfg.hdfs.datanode.dataDirs) else d; (concatMapStringsSep "," (x: "["+x.type+"]file://"+x.path) cfg.hdfs.datanode.dataDirs);
}) })
(hadoopServiceConfig { (hadoopServiceConfig {

8
nixos/modules/services/cluster/hadoop/yarn.nix
View File

@ -178,18 +178,18 @@ in
services.hadoop.gatewayRole.enable = true; services.hadoop.gatewayRole.enable = true;
services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; { services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; mkMerge [ ({
"yarn.nodemanager.local-dirs" = localDir; "yarn.nodemanager.local-dirs" = mkIf (localDir!= null) (concatStringsSep "," localDir);
"yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores; "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
"yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB; "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
"yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
"yarn.nodemanager.resource.memory-mb" = resource.memoryMB; "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
} // mkIf useCGroups { }) (mkIf useCGroups {
"yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
"yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
"yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
"yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup"; "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup";
}; })];
networking.firewall.allowedTCPPortRanges = [ networking.firewall.allowedTCPPortRanges = [
(mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;}) (mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;})

2
nixos/tests/hadoop/yarn.nix
View File

@ -19,7 +19,7 @@ import ../make-test-python.nix ({ package, ... }: {
enable = true; enable = true;
openFirewall = true; openFirewall = true;
}; };
yarnSite = options.services.hadoop.yarnSite.default // { yarnSite = {
"yarn.resourcemanager.hostname" = "resourcemanager"; "yarn.resourcemanager.hostname" = "resourcemanager";
"yarn.nodemanager.log-dirs" = "/tmp/userlogs"; "yarn.nodemanager.log-dirs" = "/tmp/userlogs";
}; };