Merge pull request #256449 from illustris/hadoop

hadoop: 3.3.5 -> 3.3.6, build container executor from source
2023-10-22 01:40:19 +02:00 · 2023-10-22 01:40:19 +02:00 · d2af1eb6ff
commit d2af1eb6ff
parent 5883b305e7 190f3903bb
6 changed files with 122 additions and 68 deletions
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@ -67,16 +67,16 @@ with lib;
    mapredSiteDefault = mkOption {
      default = {
        "mapreduce.framework.name" = "yarn";
-        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
-        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
-        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
+        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
      };
      defaultText = literalExpression ''
        {
          "mapreduce.framework.name" = "yarn";
-          "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
-          "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
-          "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
+          "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
        }
      '';
      type = types.attrsOf types.anything;
@ -154,13 +154,13 @@ with lib;
    };

    log4jProperties = mkOption {
-      default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
+      default = "${cfg.package}/etc/hadoop/log4j.properties";
      defaultText = literalExpression ''
-        "''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}/etc/hadoop/log4j.properties"
+        "''${config.${opt.package}}/etc/hadoop/log4j.properties"
      '';
      type = types.path;
      example = literalExpression ''
-        "''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
+        "''${pkgs.hadoop}/etc/hadoop/log4j.properties";
      '';
      description = lib.mdDoc "log4j.properties file added to HADOOP_CONF_DIR";
    };
--- a/nixos/modules/services/cluster/hadoop/yarn.nix
+++ b/nixos/modules/services/cluster/hadoop/yarn.nix
@ -160,7 +160,7 @@ in
          umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
          rm -rf /run/wrappers/yarn-nodemanager/ || true
          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
-          cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
+          cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
          chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
          chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
          cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
--- a/nixos/tests/hadoop/hadoop.nix
+++ b/nixos/tests/hadoop/hadoop.nix
@ -249,7 +249,7 @@ import ../make-test-python.nix ({ package, ... }: {
    assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
    client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")

-    assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
+    assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
    assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")
  '';
 })
--- a/pkgs/applications/networking/cluster/hadoop/containerExecutor.nix
+++ b/pkgs/applications/networking/cluster/hadoop/containerExecutor.nix
@ -0,0 +1,37 @@
+{ version, stdenv, fetchurl, lib, cmake, openssl, platformAttrs, ... }:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "hadoop-yarn-containerexecutor";
+  inherit version;
+
+  src = fetchurl {
+    url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}-src.tar.gz";
+    hash = platformAttrs.${stdenv.system}.srcHash;
+  };
+  sourceRoot = "hadoop-${finalAttrs.version}-src/hadoop-yarn-project/hadoop-yarn/"
+               +"hadoop-yarn-server/hadoop-yarn-server-nodemanager/src";
+
+  nativeBuildInputs = [ cmake ];
+  buildInputs = [ openssl ];
+  cmakeFlags = [ "-DHADOOP_CONF_DIR=/run/wrappers/yarn-nodemanager/etc/hadoop" ];
+
+  installPhase = ''
+    mkdir $out
+    mv target/var/empty/local/bin $out/
+  '';
+
+  meta = with lib; {
+    homepage = "https://hadoop.apache.org/";
+    description = "Framework for distributed processing of large data sets across clusters of computers";
+    license = licenses.asl20;
+
+    longDescription = ''
+        The Hadoop YARN Container Executor is a native component responsible for managing the lifecycle of containers
+        on individual nodes in a Hadoop YARN cluster. It launches, monitors, and terminates containers, ensuring that
+        resources like CPU and memory are allocated according to the policies defined in the ResourceManager.
+    '';
+
+    maintainers = with maintainers; [ illustris ];
+    platforms = filter (strings.hasSuffix "linux") (attrNames platformAttrs);
+  };
+})
--- a/pkgs/applications/networking/cluster/hadoop/default.nix
+++ b/pkgs/applications/networking/cluster/hadoop/default.nix
@ -19,6 +19,8 @@
 , nixosTests
 , sparkSupport ? true
 , spark
+, libtirpc
+, callPackage
 }:

 with lib;
@ -26,40 +28,75 @@ with lib;
 assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];

 let
-  common = { pname, platformAttrs, untarDir ? "${pname}-${version}", jdk, openssl ? null, nativeLibs ? [ ], libPatches ? "", tests }:
-    stdenv.mkDerivation rec {
-      inherit pname jdk libPatches untarDir openssl;
+  common = { pname, platformAttrs, jdk, tests }:
+    stdenv.mkDerivation (finalAttrs: {
+      inherit pname jdk;
      version = platformAttrs.${stdenv.system}.version or (throw "Unsupported system: ${stdenv.system}");
      src = fetchurl {
-        url = "mirror://apache/hadoop/common/hadoop-${version}/hadoop-${version}" + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz";
+        url = "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}"
+              + optionalString stdenv.isAarch64 "-aarch64" + ".tar.gz";
        inherit (platformAttrs.${stdenv.system}) hash;
      };
      doCheck = true;

+      # Build the container executor binary from source
+      # InstallPhase is not lazily evaluating containerExecutor for some reason
+      containerExecutor = if stdenv.isLinux then (callPackage ./containerExecutor.nix {
+        inherit (finalAttrs) version;
+        inherit platformAttrs;
+      }) else "";
+
      nativeBuildInputs = [ makeWrapper ]
-        ++ optionals (stdenv.isLinux && (nativeLibs != [ ] || libPatches != "")) [ autoPatchelfHook ];
-      buildInputs = [ openssl ] ++ nativeLibs;
+                          ++ optionals stdenv.isLinux [ autoPatchelfHook ];
+      buildInputs = optionals stdenv.isLinux [ stdenv.cc.cc.lib openssl protobuf zlib snappy libtirpc ];

      installPhase = ''
-        mkdir -p $out/{lib/${untarDir}/conf,bin,lib}
-        mv * $out/lib/${untarDir}
+        mkdir $out
+        mv * $out/
      '' + optionalString stdenv.isLinux ''
-        # All versions need container-executor, but some versions can't use autoPatchelf because of broken SSL versions
-        patchelf --set-interpreter ${glibc.out}/lib64/ld-linux-x86-64.so.2 $out/lib/${untarDir}/bin/container-executor
-      '' + ''
-        for n in $(find $out/lib/${untarDir}/bin -type f ! -name "*.*"); do
-          makeWrapper "$n" "$out/bin/$(basename $n)"\
-            --set-default JAVA_HOME ${jdk.home}\
-            --set-default HADOOP_HOME $out/lib/${untarDir}\
-            --run "test -d /etc/hadoop-conf && export HADOOP_CONF_DIR=\''${HADOOP_CONF_DIR-'/etc/hadoop-conf/'}"\
-            --set-default HADOOP_CONF_DIR $out/lib/${untarDir}/etc/hadoop/\
-            --prefix PATH : "${makeBinPath [ bash coreutils which]}"\
-            --prefix JAVA_LIBRARY_PATH : "${makeLibraryPath buildInputs}"
+        for n in $(find ${finalAttrs.containerExecutor}/bin -type f); do
+          ln -sf "$n" $out/bin
        done
-      '' + optionalString sparkSupport ''
+
+        # these libraries are loaded at runtime by the JVM
+        ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/native/libsasl2.so.2
+        ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/native/
+        ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/native/
+        ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/native/
+        ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/native/
+        ln -s ${getLib snappy}/lib/libsnappy.so.1 $out/lib/native/
+
+        # libjvm.so is in different paths for java 8 and 11
+        # libnativetask.so in hadooop 3 and libhdfs.so in hadoop 2 depend on it
+        find $out/lib/native/ -name 'libnativetask.so*' -o -name 'libhdfs.so*' | \
+          xargs -n1 patchelf --add-rpath $(dirname $(find ${finalAttrs.jdk.home} -name libjvm.so | head -n1))
+
+        # NixOS/nixpkgs#193370
+        # This workaround is needed to use protobuf 3.19
+        # hadoop 3.3+ depends on protobuf 3.18, 3.2 depends on 3.8
+        find $out/lib/native -name 'libhdfspp.so*' | \
+          xargs -r -n1 patchelf --replace-needed libprotobuf.so.${
+            if (versionAtLeast finalAttrs.version "3.3") then "18"
+            else "8"
+          } libprotobuf.so
+
+        patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \
+          $out/lib/native/{libhdfs{pp,}.so*,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}}
+
+      '' + ''
+        for n in $(find $out/bin -type f ! -name "*.*"); do
+          wrapProgram "$n"\
+            --set-default JAVA_HOME ${finalAttrs.jdk.home}\
+            --set-default HADOOP_HOME $out/\
+            --run "test -d /etc/hadoop-conf && export HADOOP_CONF_DIR=\''${HADOOP_CONF_DIR-'/etc/hadoop-conf/'}"\
+            --set-default HADOOP_CONF_DIR $out/etc/hadoop/\
+            --prefix PATH : "${makeBinPath [ bash coreutils which]}"\
+            --prefix JAVA_LIBRARY_PATH : "${makeLibraryPath finalAttrs.buildInputs}"
+        done
+      '' + (optionalString sparkSupport ''
        # Add the spark shuffle service jar to YARN
-        cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/lib/${untarDir}/share/hadoop/yarn/
-      '' + libPatches;
+        cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/share/hadoop/yarn/
+      '');

      passthru = { inherit tests; };

@ -83,7 +120,7 @@ let
        maintainers = with maintainers; [ illustris ];
        platforms = attrNames platformAttrs;
      } (attrByPath [ stdenv.system "meta" ] {} platformAttrs);
-    };
+    });
 in
 {
  # Different version of hadoop support different java runtime versions
@ -91,48 +128,29 @@ in
  hadoop_3_3 = common rec {
    pname = "hadoop";
    platformAttrs = rec {
-        x86_64-linux = {
-          version = "3.3.5";
-          hash = "sha256-RG4FypL6I6YGF6ixeUbe3kcoGvFQQEFhfLfV9i50JSo=";
-        };
-        x86_64-darwin = x86_64-linux;
-        aarch64-linux = {
-          version = "3.3.5";
-          hash = "sha256-qcKjbE881isauWBxIv+NY0UFbYit704/Re8Kdl6x1LA=";
-        };
-        aarch64-darwin = aarch64-linux;
+      x86_64-linux = {
+        version = "3.3.6";
+        hash = "sha256-9RlQWcDUECrap//xf3sqhd+Qa8tuGZSHFjGfmXhkGgQ=";
+        srcHash = "sha256-4OEsVhBNV9CJ+PN4FgCduUCVA9/el5yezSCZ6ko3+bU=";
+      };
+      x86_64-darwin = x86_64-linux;
+      aarch64-linux = x86_64-linux // {
+        hash = "sha256-5Lv2uA72BJEva5v2yncyPe5gKNCNOPNsoHffVt6KXQ0=";
+      };
+      aarch64-darwin = aarch64-linux;
    };
-    untarDir = "${pname}-${platformAttrs.${stdenv.system}.version}";
    jdk = jdk11_headless;
-    inherit openssl;
    # TODO: Package and add Intel Storage Acceleration Library
-    nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy ];
-    libPatches = ''
-      ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${untarDir}/lib/native/libsasl2.so.2
-      ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${untarDir}/lib/native/
-      ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${untarDir}/lib/native/
-      ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${untarDir}/lib/native/
-      ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${untarDir}/lib/native/
-    '' + optionalString stdenv.isLinux ''
-      # libjvm.so for Java >=11
-      patchelf --add-rpath ${jdk.home}/lib/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0
-      # Java 8 has libjvm.so at a different path
-      patchelf --add-rpath ${jdk.home}/jre/lib/amd64/server $out/lib/${untarDir}/lib/native/libnativetask.so.1.0.0
-      # NixOS/nixpkgs#193370
-      # This workaround is needed to use protobuf 3.19
-      patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${untarDir}/lib/native/libhdfspp.so
-    '';
    tests = nixosTests.hadoop;
  };
-  hadoop_3_2 = common rec {
+  hadoop_3_2 = common {
    pname = "hadoop";
    platformAttrs.x86_64-linux = {
      version = "3.2.4";
      hash = "sha256-qt2gpMr+NHuiVR+/zFRzRyRKG725/ZNBIM69z9J9wNw=";
+      srcHash = "sha256-F9nGD3mZZ1eJf3Ec3AJGE9YBcL/HiagskcdKQhCn/sw=";
    };
    jdk = jdk8_headless;
-    # not using native libs because of broken openssl_1_0_2 dependency
-    # can be manually overridden
    tests = nixosTests.hadoop_3_2;
  };
  hadoop2 = common rec {
@ -140,6 +158,7 @@ in
    platformAttrs.x86_64-linux = {
      version = "2.10.2";
      hash = "sha256-xhA4zxqIRGNhIeBnJO9dLKf/gx/Bq+uIyyZwsIafEyo=";
+      srcHash = "sha256-ucxCyXiJo8aL6aNMhZgKEbn8sGKOoMPVREbMGSfSdAI=";
    };
    jdk = jdk8_headless;
    tests = nixosTests.hadoop2;
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -17702,9 +17702,7 @@ with pkgs;

  groovy = callPackage ../development/interpreters/groovy { };

-  inherit (callPackages ../applications/networking/cluster/hadoop {
-    openssl = openssl_1_1;
-  })
+  inherit (callPackages ../applications/networking/cluster/hadoop {})
    hadoop_3_3
    hadoop_3_2
    hadoop2;