Merge pull request #327444 from doronbehar/pkg/mpi4py

python312Packages.mpi4py: 3.1.6 -> 3.1.6-unstable-2024-07-08; rewrite

pkgs/development/python-modules/aplpy/default.nix

@@ -81,5 +81,9 @@ buildPythonPackage rec {
     homepage = "http://aplpy.github.io";
     license = licenses.mit;
     maintainers = with maintainers; [ smaret ];
+    # Tests fail due to expired matplotlib deprecations:
+    #
+    # > (`matplotlib.cm' has no attribute 'get_cmap'`)
+    broken = true;
   };
 }

pkgs/development/python-modules/h5py/default.nix

@@ -4,14 +4,15 @@
   buildPythonPackage,
   pythonOlder,
   setuptools,
-  wheel,
   numpy,
   hdf5,
+  pythonRelaxDepsHook,
   cython_0,
   pkgconfig,
   mpi4py ? null,
   openssh,
   pytestCheckHook,
+  pytest-mpi,
   cached-property,
 }:
 
@@ -39,11 +40,15 @@ buildPythonPackage rec {
     ./mpi4py-requirement.patch
   ];
 
-  # avoid strict pinning of numpy
+  # avoid strict pinning of numpy, can't be replaced with pythonRelaxDepsHook,
+  # see: https://github.com/NixOS/nixpkgs/issues/327941
   postPatch = ''
     substituteInPlace pyproject.toml \
       --replace-fail "numpy >=2.0.0rc1" "numpy"
   '';
+  pythonRelaxDeps = [
+    "mpi4py"
+  ];
 
   HDF5_DIR = "${hdf5}";
   HDF5_MPI = if mpiSupport then "ON" else "OFF";
@@ -57,11 +62,10 @@ buildPythonPackage rec {
   preBuild = lib.optionalString mpiSupport "export CC=${lib.getDev mpi}/bin/mpicc";
 
   nativeBuildInputs = [
+    pythonRelaxDepsHook
     cython_0
-    numpy
     pkgconfig
     setuptools
-    wheel
   ];
 
   buildInputs = [ hdf5 ] ++ lib.optional mpiSupport mpi;
@@ -74,20 +78,27 @@ buildPythonPackage rec {
     ]
     ++ lib.optionals (pythonOlder "3.8") [ cached-property ];
 
-  # tests now require pytest-mpi, which isn't available and difficult to package
-  doCheck = false;
   nativeCheckInputs = [
     pytestCheckHook
+    pytest-mpi
     openssh
   ];
+  # https://github.com/NixOS/nixpkgs/issues/255262
+  preCheck = ''
+    cd $out
+  '';
+  # For some reason these fail when mpi support is enabled, due to concurrent
+  # writings. There are a few open issues about this in the bug tracker, but
+  # not related to the tests.
+  disabledTests = lib.optionals mpiSupport [ "TestPageBuffering" ];
 
   pythonImportsCheck = [ "h5py" ];
 
-  meta = with lib; {
+  meta = {
     changelog = "https://github.com/h5py/h5py/blob/${version}/docs/whatsnew/${lib.versions.majorMinor version}.rst";
     description = "Pythonic interface to the HDF5 binary data format";
     homepage = "http://www.h5py.org/";
-    license = licenses.bsd3;
-    maintainers = [ ];
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ doronbehar ];
   };
 }

pkgs/development/python-modules/hoomd-blue/default.nix

@@ -66,5 +66,8 @@ buildPythonPackage rec {
     license = licenses.bsdOriginal;
     platforms = [ "x86_64-linux" ];
     maintainers = [ ];
+    # Has compilation errors since some dependencies got updated, will probably
+    # be fixed if updated by itself to the latest version.
+    broken = true;
   };
 }

pkgs/development/python-modules/mne-python/default.nix

@@ -73,6 +73,11 @@ buildPythonPackage rec {
     export MNE_SKIP_NETWORK_TESTS=1
   '';
 
+  disabledTests = [
+    # Fails due to changes in Numpy types
+    "mne.stats._adjacency.combine_adjacency"
+  ];
+
   pythonImportsCheck = [ "mne" ];
 
   meta = with lib; {

pkgs/development/python-modules/mpi4py/default.nix

@@ -1,61 +1,62 @@
 {
   lib,
-  fetchPypi,
-  python,
+  fetchFromGitHub,
   buildPythonPackage,
+  cython,
+  setuptools,
   mpi,
-  mpiCheckPhaseHook,
   openssh,
+  pytestCheckHook,
+  mpiCheckPhaseHook,
 }:
 
 buildPythonPackage rec {
   pname = "mpi4py";
-  version = "3.1.6";
-  format = "setuptools";
+  # See https://github.com/mpi4py/mpi4py/issues/386 . Part of the changes since
+  # the last release include Python 3.12 fixes.
+  version = "3.1.6-unstable-2024-07-08";
+  pyproject = true;
 
-  src = fetchPypi {
-    inherit pname version;
-    hash = "sha256-yPpiXg+SsILvlVv7UvGfpmkdKSc9fXETXSlaoUPe5ss=";
+  src = fetchFromGitHub {
+    repo = "mpi4py";
+    owner = "mpi4py";
+    rev = "e9a59719bbce1b9c351e1e30ecd3be3b459e97cd";
+    hash = "sha256-C/nidWGr8xsLV73u7HRtnXoQgYmoRJkD45DFrdXXTPI=";
   };
 
+  build-system = [
+    cython
+    setuptools
+    mpi
+  ];
+  dependencies = [
+    mpi
+  ];
+
+  __darwinAllowLocalNetworking = true;
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    openssh
+    mpiCheckPhaseHook
+  ];
+  # Most tests pass, (besides `test_spawn.py`), but when reaching ~80% tests
+  # progress, an orted process hangs and the tests don't finish. This issue is
+  # probably due to the sandbox.
+  doCheck = false;
+  disabledTestPaths = [
+    # Almost all tests in this file fail (TODO: Report about this upstream..)
+    "test/test_spawn.py"
+  ];
+
   passthru = {
     inherit mpi;
   };
 
-  postPatch = ''
-    substituteInPlace test/test_spawn.py --replace \
-                      "unittest.skipMPI('openmpi(<3.0.0)')" \
-                      "unittest.skipMPI('openmpi')"
-  '';
-
-  configurePhase = "";
-
-  installPhase = ''
-    mkdir -p "$out/${python.sitePackages}"
-    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-
-    ${python}/bin/${python.executable} setup.py install \
-      --install-lib=$out/${python.sitePackages} \
-      --prefix="$out"
-
-    # --install-lib:
-    # sometimes packages specify where files should be installed outside the usual
-    # python lib prefix, we override that back so all infrastructure (setup hooks)
-    # work as expected
-  '';
-
-  nativeBuildInputs = [ mpi ];
-
-  __darwinAllowLocalNetworking = true;
-
-  nativeCheckInputs = [
-    openssh
-    mpiCheckPhaseHook
-  ];
-
-  meta = with lib; {
+  meta = {
     description = "Python bindings for the Message Passing Interface standard";
     homepage = "https://github.com/mpi4py/mpi4py";
-    license = licenses.bsd2;
+    license = lib.licenses.bsd2;
+    maintainers = with lib.maintainers; [ doronbehar ];
   };
 }

pkgs/development/python-modules/mpi4py/tests.patch

@@ -1,13 +0,0 @@
-diff --git i/test/test_dl.py w/test/test_dl.py
-index a3211a3..9d25569 100644
---- i/test/test_dl.py
-+++ w/test/test_dl.py
-@@ -12,7 +12,7 @@ class TestDL(unittest.TestCase):
-         if sys.platform == 'darwin':
-             libm = 'libm.dylib'
-         else:
--            libm = 'libm.so'
-+            libm = 'libm.so.6'
- 
-         handle = dl.dlopen(libm, dl.RTLD_LOCAL|dl.RTLD_LAZY)
-         self.assertTrue(handle != 0)

pkgs/development/python-modules/nitime/default.nix

@@ -27,11 +27,11 @@ buildPythonPackage rec {
     hash = "sha256-4Ie8fuk9CKdn/64TsCfN2No2dU16ICpBRWYerqqF0/0=";
   };
 
-  # Upstream wants to build against the oldest version of numpy possible, but
-  # we only want to build against the most recent version.
+  # Upstream wants to build against the newest version of numpy possible, but
+  # we only want to build against our default numpy.
   postPatch = ''
     substituteInPlace pyproject.toml \
-      --replace "numpy==" "numpy>="
+      --replace-fail "numpy>=2.0.0rc1,<3" "numpy"
   '';
 
   nativeBuildInputs = [

pkgs/development/python-modules/objax/default.nix

@@ -77,5 +77,7 @@ buildPythonPackage rec {
     changelog = "https://github.com/google/objax/releases/tag/v${version}";
     license = licenses.asl20;
     maintainers = with maintainers; [ ndl ];
+    # Tests test_syncbn_{0,1,2}d and other tests from tests/parallel.py fail
+    broken = true;
   };
 }

pkgs/development/python-modules/outlines/default.nix

@@ -57,5 +57,7 @@ buildPythonPackage rec {
     homepage = "https://github.com/outlines-dev/outlines";
     license = licenses.asl20;
     maintainers = with maintainers; [ lach ];
+    # Missing dependencies since the last update
+    broken = true;
   };
 }

pkgs/development/python-modules/pymatgen/default.nix

@@ -112,6 +112,6 @@ buildPythonPackage rec {
     changelog = "https://github.com/materialsproject/pymatgen/releases/tag/v${version}";
     license = licenses.mit;
     maintainers = with maintainers; [ psyanticy ];
-    broken = stdenv.isDarwin; # tests segfault. that's bad.
+    broken = true; # tests segfault. that's bad.
   };
 }

pkgs/development/python-modules/pyprecice/default.nix

@@ -1,6 +1,8 @@
 {
   lib,
   buildPythonPackage,
+  setuptools,
+  pip,
   cython,
   fetchFromGitHub,
   mpi4py,
@@ -12,8 +14,8 @@
 
 buildPythonPackage rec {
   pname = "pyprecice";
-  version = "3.1.0";
-  format = "setuptools";
+  version = "3.1.1";
+  pyproject = true;
 
   disabled = pythonOlder "3.7";
 
@@ -21,10 +23,12 @@ buildPythonPackage rec {
     owner = "precice";
     repo = "python-bindings";
     rev = "refs/tags/v${version}";
-    hash = "sha256-5K6oVBhR6mBdkyOb/Ec0qg9x63tkoTnLIrE8dz8oCtc=";
+    hash = "sha256-qeDWj03Uo0Kf75MN0eI+DBwa94v7GRe6+FKYtVOM6vs=";
   };
 
   nativeBuildInputs = [
+    setuptools
+    pip
     cython
     pkgconfig
   ];

pkgs/development/python-modules/pyscf/default.nix

@@ -86,6 +86,7 @@ buildPythonPackage rec {
     "test_veff"
     "test_collinear_kgks_gga"
     "test_libxc_gga_deriv4"
+    "test_sacasscf_grad"
   ];
 
   pytestFlagsArray = [
@@ -94,6 +95,8 @@ buildPythonPackage rec {
     "--ignore-glob=*_slow.*py"
     "--ignore-glob=*_kproxy_.*py"
     "--ignore-glob=test_proxy.py"
+    "--ignore-glob=pyscf/nac/test/test_sacasscf.py"
+    "--ignore-glob=pyscf/grad/test/test_casscf.py"
   ];
 
   meta = with lib; {

pkgs/development/python-modules/pytest-mpi/default.nix

@@ -0,0 +1,56 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, setuptools
+, sybil
+, pytest
+, pytestCheckHook
+, mpi
+, mpi4py
+}:
+
+buildPythonPackage rec {
+  pname = "pytest-mpi";
+  version = "0.6";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "aragilar";
+    repo = "pytest-mpi";
+    rev = "v${version}";
+    hash = "sha256-m3HTGLoPnYeg0oeIA1nzTzch7FtkuXTYpox4rRgo5MU=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+  buildInputs = [
+    # Don't propagate it to let a different pytest version be used if needed
+    pytest
+  ];
+  dependencies = [
+    sybil
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+    mpi
+    mpi4py
+  ];
+  # Tests cause the Python interpreter to crash from some reason, a hard issue
+  # to debug. (TODO: discuss this with upstream)
+  doCheck = false;
+  pytestFlagsArray = [
+    # https://github.com/aragilar/pytest-mpi/issues/4#issuecomment-634614337
+    "-p" "pytester"
+  ];
+
+  pythonImportsCheck = [ "pytest_mpi" ];
+
+  meta = {
+    description = "Pytest plugin for working with MPI";
+    homepage = "https://github.com/aragilar/pytest-mpi";
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ doronbehar ];
+  };
+}

pkgs/tools/misc/diffoscope/default.nix

@@ -267,6 +267,8 @@ python.pkgs.buildPythonApplication rec {
 
     # Fails because it fails to determine llvm version
     "test_item3_deflate_llvm_bitcode"
+    # Fails ever since an update to imagemagick
+    "test_has_visuals"
   ] ++ lib.optionals stdenv.isDarwin [
     # Disable flaky tests on Darwin
     "test_non_unicode_filename"

pkgs/top-level/python-packages.nix

@@ -12452,6 +12452,8 @@ self: super: with self; {
 
   pytest-mockservers = callPackage ../development/python-modules/pytest-mockservers { };
 
+  pytest-mpi = callPackage ../development/python-modules/pytest-mpi { };
+
   pytest-mpl = callPackage ../development/python-modules/pytest-mpl { };
 
   pytest-mypy = callPackage ../development/python-modules/pytest-mypy { };