From ddb161e01a43b60e2385af4a3eaa3c8e877aa14a Mon Sep 17 00:00:00 2001 From: Alex Biehl Date: Wed, 20 May 2020 12:15:56 +0200 Subject: [PATCH 1/4] python3Packages.skein: init at 0.8.0 --- .../python-modules/skein/default.nix | 101 ++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 103 insertions(+) create mode 100644 pkgs/development/python-modules/skein/default.nix diff --git a/pkgs/development/python-modules/skein/default.nix b/pkgs/development/python-modules/skein/default.nix new file mode 100644 index 000000000000..82e4ba63ba68 --- /dev/null +++ b/pkgs/development/python-modules/skein/default.nix @@ -0,0 +1,101 @@ +{ autoPatchelfHook +, buildPythonPackage +, coreutils +, fetchPypi +, jre +, lib +, maven +, protobuf +, pythonPackages +, stdenv +}: + +buildPythonPackage rec { + pname = "skein"; + version = "0.8.0"; + + src = fetchPypi { + inherit pname version; + sha256 = "0nb64p1hzshgi1kfc2jx1v9vn8b0wzs50460wfra3fsxh0ap66ab"; + }; + + skeinRepo = stdenv.mkDerivation rec { + name = "${pname}-${version}-maven-repo"; + + inherit src; + + nativeBuildInputs = [ maven ] ++ lib.optional stdenv.isLinux autoPatchelfHook; + + buildPhase = ""; + + installPhase = '' + mkdir -p $out + + archs="${ + if stdenv.isLinux + then "linux-x86_32 linux-x86_64" + else "osx-x86_64" + }" + + for arch in $archs + do + mvn -Dmaven.repo.local=$out dependency:get -Dartifact=com.google.protobuf:protoc:3.0.0:exe:$arch + mvn -Dmaven.repo.local=$out dependency:get -Dartifact=io.grpc:protoc-gen-grpc-java:1.16.0:exe:$arch + done + + if ${ lib.boolToString stdenv.isLinux } + then + autoPatchelf $out + fi + + # We have to use maven package here as dependency:go-offline doesn't + # fetch every required jar. + mvn -f java/pom.xml -Dmaven.repo.local=$out package + + rm $(find $out -name _remote.repositories) + rm $(find $out -name resolver-status.properties) + ''; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = if stdenv.isLinux + then "12f0q3444qw6y4f6qsa9540a0fz4cgi844zzi8z1phqn3k4dnl6v" + else "0bjbwiv17cary1isxca0m2hsvgs1i5fh18z247h1hky73lnhbrz8"; + + } // lib.optionalAttrs stdenv.isLinux { dontAutoPatchelf = true; }; + + # skein wants to compile a .jar file for Hadoop interaction. We build + # it separately as otherwise mvn is called from setup.py and we have + # no good way to inject the repository in there. + skeinJar = stdenv.mkDerivation rec { + name = "${pname}-${version}.jar"; + + inherit src; + + nativeBuildInputs = [ maven ]; + + buildPhase = '' + mvn --offline -f java/pom.xml package -Dmaven.repo.local="${skeinRepo}" -Dskein.version=${version} -Dversion=${version} + ''; + + installPhase = '' + # Making sure skein.jar exists skips the maven build in setup.py + mv java/target/skein-*.jar $out + ''; + }; + + propagatedBuildInputs = with pythonPackages; [ cryptography grpcio grpcio-tools jupyter pytest pyyaml requests jre ]; + + preBuild = '' + # Making sure skein.jar exists skips the maven build in setup.py + mkdir -p skein/java + ln -s ${skeinJar} skein/java/skein.jar + ''; + + meta = with stdenv.lib; { + homepage = "https://jcristharif.com/skein"; + description = "A tool and library for easily deploying applications on Apache YARN"; + license = licenses.bsd3; + }; + +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 885622b9be95..8d99e2363a1d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6338,6 +6338,8 @@ in { python_statsd = callPackage ../development/python-modules/python_statsd { }; + skein = disabledIf isPy27 (callPackage ../development/python-modules/skein { }); + stompclient = callPackage ../development/python-modules/stompclient { }; subdownloader = callPackage ../development/python-modules/subdownloader { }; From 6200d2f8b235bbc2adb9db45e54cae02a407a87e Mon Sep 17 00:00:00 2001 From: Alex Biehl Date: Fri, 29 May 2020 07:35:25 +0200 Subject: [PATCH 2/4] Add alexbiehl as maintainer --- maintainers/maintainer-list.nix | 6 ++++++ pkgs/development/python-modules/skein/default.nix | 1 + 2 files changed, 7 insertions(+) diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 3a8c5e5a76b4..162441b23f17 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -313,6 +313,12 @@ githubId = 2387841; name = "Alexander Bakker"; }; + alexbiehl = { + email = "alexbiehl@gmail.com"; + github = "alexbiehl"; + githubId = 1876617; + name = "Alex Biehl"; + }; alexchapman = { email = "alex@farfromthere.net"; github = "AJChapman"; diff --git a/pkgs/development/python-modules/skein/default.nix b/pkgs/development/python-modules/skein/default.nix index 82e4ba63ba68..92de47e78847 100644 --- a/pkgs/development/python-modules/skein/default.nix +++ b/pkgs/development/python-modules/skein/default.nix @@ -96,6 +96,7 @@ buildPythonPackage rec { homepage = "https://jcristharif.com/skein"; description = "A tool and library for easily deploying applications on Apache YARN"; license = licenses.bsd3; + maintainers = with maintainers; [ alexbiehl ]; }; } From 1e4308dfd429d7a20f8b1319e63d25a7493cf771 Mon Sep 17 00:00:00 2001 From: Alex Biehl Date: Fri, 29 May 2020 07:54:48 +0200 Subject: [PATCH 3/4] Split out monlithic derivation into multiple files --- .../python-modules/skein/default.nix | 82 ++----------------- .../python-modules/skein/skeinjar.nix | 21 +++++ .../python-modules/skein/skeinrepo.nix | 46 +++++++++++ 3 files changed, 75 insertions(+), 74 deletions(-) create mode 100644 pkgs/development/python-modules/skein/skeinjar.nix create mode 100644 pkgs/development/python-modules/skein/skeinrepo.nix diff --git a/pkgs/development/python-modules/skein/default.nix b/pkgs/development/python-modules/skein/default.nix index 92de47e78847..dd98bdddc3a2 100644 --- a/pkgs/development/python-modules/skein/default.nix +++ b/pkgs/development/python-modules/skein/default.nix @@ -1,93 +1,27 @@ -{ autoPatchelfHook -, buildPythonPackage -, coreutils +{ buildPythonPackage +, callPackage , fetchPypi , jre , lib -, maven -, protobuf , pythonPackages , stdenv }: - -buildPythonPackage rec { +let pname = "skein"; version = "0.8.0"; - src = fetchPypi { inherit pname version; sha256 = "0nb64p1hzshgi1kfc2jx1v9vn8b0wzs50460wfra3fsxh0ap66ab"; }; - - skeinRepo = stdenv.mkDerivation rec { - name = "${pname}-${version}-maven-repo"; - - inherit src; - - nativeBuildInputs = [ maven ] ++ lib.optional stdenv.isLinux autoPatchelfHook; - - buildPhase = ""; - - installPhase = '' - mkdir -p $out - - archs="${ - if stdenv.isLinux - then "linux-x86_32 linux-x86_64" - else "osx-x86_64" - }" - - for arch in $archs - do - mvn -Dmaven.repo.local=$out dependency:get -Dartifact=com.google.protobuf:protoc:3.0.0:exe:$arch - mvn -Dmaven.repo.local=$out dependency:get -Dartifact=io.grpc:protoc-gen-grpc-java:1.16.0:exe:$arch - done - - if ${ lib.boolToString stdenv.isLinux } - then - autoPatchelf $out - fi - - # We have to use maven package here as dependency:go-offline doesn't - # fetch every required jar. - mvn -f java/pom.xml -Dmaven.repo.local=$out package - - rm $(find $out -name _remote.repositories) - rm $(find $out -name resolver-status.properties) - ''; - - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = if stdenv.isLinux - then "12f0q3444qw6y4f6qsa9540a0fz4cgi844zzi8z1phqn3k4dnl6v" - else "0bjbwiv17cary1isxca0m2hsvgs1i5fh18z247h1hky73lnhbrz8"; - - } // lib.optionalAttrs stdenv.isLinux { dontAutoPatchelf = true; }; - - # skein wants to compile a .jar file for Hadoop interaction. We build - # it separately as otherwise mvn is called from setup.py and we have - # no good way to inject the repository in there. - skeinJar = stdenv.mkDerivation rec { - name = "${pname}-${version}.jar"; - - inherit src; - - nativeBuildInputs = [ maven ]; - - buildPhase = '' - mvn --offline -f java/pom.xml package -Dmaven.repo.local="${skeinRepo}" -Dskein.version=${version} -Dversion=${version} - ''; - - installPhase = '' - # Making sure skein.jar exists skips the maven build in setup.py - mv java/target/skein-*.jar $out - ''; - }; + skeinJar = callPackage ./skeinjar.nix { inherit src version; }; +in +buildPythonPackage rec { + inherit pname version src; propagatedBuildInputs = with pythonPackages; [ cryptography grpcio grpcio-tools jupyter pytest pyyaml requests jre ]; preBuild = '' - # Making sure skein.jar exists skips the maven build in setup.py + # Ensure skein.jar exists skips the maven build in setup.py mkdir -p skein/java ln -s ${skeinJar} skein/java/skein.jar ''; diff --git a/pkgs/development/python-modules/skein/skeinjar.nix b/pkgs/development/python-modules/skein/skeinjar.nix new file mode 100644 index 000000000000..c1a0d8a57515 --- /dev/null +++ b/pkgs/development/python-modules/skein/skeinjar.nix @@ -0,0 +1,21 @@ +{ callPackage, stdenv, maven, src, version }: + +let + skeinRepo = callPackage ./skeinrepo.nix { inherit src version; }; +in +stdenv.mkDerivation rec { + name = "skein-${version}.jar"; + + inherit src; + + nativeBuildInputs = [ maven ]; + + buildPhase = '' + mvn --offline -f java/pom.xml package -Dmaven.repo.local="${skeinRepo}" -Dskein.version=${version} -Dversion=${version} + ''; + + installPhase = '' + # Making sure skein.jar exists skips the maven build in setup.py + mv java/target/skein-*.jar $out + ''; +} diff --git a/pkgs/development/python-modules/skein/skeinrepo.nix b/pkgs/development/python-modules/skein/skeinrepo.nix new file mode 100644 index 000000000000..85b4f19c8f73 --- /dev/null +++ b/pkgs/development/python-modules/skein/skeinrepo.nix @@ -0,0 +1,46 @@ +{ autoPatchelfHook, lib, maven, stdenv, src, version }: + +stdenv.mkDerivation rec { + name = "skein-${version}-maven-repo"; + + inherit src; + + nativeBuildInputs = [ maven ] ++ lib.optional stdenv.isLinux autoPatchelfHook; + + doBuild = false; + + installPhase = '' + mkdir -p $out + + archs="${ + if stdenv.isLinux + then "linux-x86_32 linux-x86_64" + else "osx-x86_64" + }" + + for arch in $archs + do + mvn -Dmaven.repo.local=$out dependency:get -Dartifact=com.google.protobuf:protoc:3.0.0:exe:$arch + mvn -Dmaven.repo.local=$out dependency:get -Dartifact=io.grpc:protoc-gen-grpc-java:1.16.0:exe:$arch + done + + if ${ lib.boolToString stdenv.isLinux } + then + autoPatchelf $out + fi + + # We have to use maven package here as dependency:go-offline doesn't + # fetch every required jar. + mvn -f java/pom.xml -Dmaven.repo.local=$out package + + rm $(find $out -name _remote.repositories) + rm $(find $out -name resolver-status.properties) + ''; + + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = if stdenv.isLinux + then "12f0q3444qw6y4f6qsa9540a0fz4cgi844zzi8z1phqn3k4dnl6v" + else "0bjbwiv17cary1isxca0m2hsvgs1i5fh18z247h1hky73lnhbrz8"; + +} // lib.optionalAttrs stdenv.isLinux { dontAutoPatchelf = true; } From a5bdbb895d9c34929a1d7745e615023f30f76fad Mon Sep 17 00:00:00 2001 From: Alexander Biehl Date: Fri, 5 Jun 2020 12:19:51 +0200 Subject: [PATCH 4/4] Apply @jonringers suggestions Co-authored-by: Jon --- pkgs/development/python-modules/skein/default.nix | 3 +++ pkgs/top-level/python-packages.nix | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/skein/default.nix b/pkgs/development/python-modules/skein/default.nix index dd98bdddc3a2..630c4370a87d 100644 --- a/pkgs/development/python-modules/skein/default.nix +++ b/pkgs/development/python-modules/skein/default.nix @@ -1,11 +1,13 @@ { buildPythonPackage , callPackage , fetchPypi +, isPy27 , jre , lib , pythonPackages , stdenv }: + let pname = "skein"; version = "0.8.0"; @@ -17,6 +19,7 @@ let in buildPythonPackage rec { inherit pname version src; + disabled = isPy27; propagatedBuildInputs = with pythonPackages; [ cryptography grpcio grpcio-tools jupyter pytest pyyaml requests jre ]; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8d99e2363a1d..7021dd612d1e 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6338,7 +6338,7 @@ in { python_statsd = callPackage ../development/python-modules/python_statsd { }; - skein = disabledIf isPy27 (callPackage ../development/python-modules/skein { }); + skein = callPackage ../development/python-modules/skein { }; stompclient = callPackage ../development/python-modules/stompclient { };