From a122884cb3709ef64c694d2658127a50dba2524b Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 2 Nov 2023 18:18:35 +0000 Subject: [PATCH 1/5] opencv4: cuda: add a libstdc++ integration test --- pkgs/development/libraries/opencv/4.x.nix | 5 ++++- .../libraries/opencv/libstdcxx-test.nix | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 pkgs/development/libraries/opencv/libstdcxx-test.nix diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index 68099a57c63f..ab7eb5c3ca0a 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -495,8 +495,11 @@ stdenv.mkDerivation { opencv4-tests = callPackage ./tests.nix { inherit enableGStreamer enableGtk2 enableGtk3 runAccuracyTests runPerformanceTests testDataSrc; inherit opencv4; - }; }; + } + // lib.optionalAttrs (enableCuda) { + no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { }; + }; } // lib.optionalAttrs enablePython { pythonPath = [ ]; }; meta = with lib; { diff --git a/pkgs/development/libraries/opencv/libstdcxx-test.nix b/pkgs/development/libraries/opencv/libstdcxx-test.nix new file mode 100644 index 000000000000..aab1101883c2 --- /dev/null +++ b/pkgs/development/libraries/opencv/libstdcxx-test.nix @@ -0,0 +1,17 @@ +{ python3Packages, runCommand }: + +runCommand "${python3Packages.opencv4.pname}-libstdcxx-test" +{ + nativeBuildInputs = [ + (python3Packages.python.withPackages (ps: with ps; [ + (opencv4.override { enableCuda = true; }) + scikit-image + ])) + ]; +} '' + python << EOF + import cv2 + from skimage.transform import pyramid_reduce + EOF + touch $out +'' From 4735963a243ff275ad1ff2aef559268932e04fe9 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 2 Nov 2023 18:21:41 +0000 Subject: [PATCH 2/5] opencv4: cuda: fix libstdc++ errors --- pkgs/development/libraries/opencv/4.x.nix | 55 +++++++++++------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index ab7eb5c3ca0a..87b116f92eb0 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -83,11 +83,16 @@ , Accelerate , bzip2 , callPackage -}: +}@inputs: let version = "4.7.0"; + # It's necessary to consistently use backendStdenv when building with CUDA + # support, otherwise we get libstdc++ errors downstream + stdenv = throw "Use effectiveStdenv instead"; + effectiveStdenv = if enableCuda then cudaPackages.backendStdenv else inputs.stdenv; + src = fetchFromGitHub { owner = "opencv"; repo = "opencv"; @@ -121,11 +126,11 @@ let sha256 = "1msbkc3zixx61rcg6a04i1bcfhw1phgsrh93glq1n80hgsk3nbjq"; } + "/ippicv"; files = let name = platform: "ippicv_2019_${platform}_general_20180723.tgz"; in - if stdenv.hostPlatform.system == "x86_64-linux" then + if effectiveStdenv.hostPlatform.system == "x86_64-linux" then { ${name "lnx_intel64"} = "c0bd78adb4156bbf552c1dfe90599607"; } - else if stdenv.hostPlatform.system == "i686-linux" then + else if effectiveStdenv.hostPlatform.system == "i686-linux" then { ${name "lnx_ia32"} = "4f38432c30bfd6423164b7a24bbc98a0"; } - else if stdenv.hostPlatform.system == "x86_64-darwin" then + else if effectiveStdenv.hostPlatform.system == "x86_64-darwin" then { ${name "mac_intel64"} = "fe6b2bb75ae0e3f19ad3ae1a31dfa4a2"; } else throw "ICV is not available for this platform (or not yet supported by this package)"; @@ -232,10 +237,11 @@ let #https://github.com/xianyi/OpenBLAS/wiki/Faq/4bded95e8dc8aadc70ce65267d1093ca7bdefc4c#multi-threaded openblas_ = blas.provider.override { singleThreaded = true; }; - inherit (cudaPackages) backendStdenv cudaFlags cudaVersion; + inherit (cudaPackages) cudaFlags cudaVersion; inherit (cudaFlags) cudaCapabilities; cuda-common-redist = with cudaPackages; [ + cuda_cudart cuda_cccl # libnpp # npp.h ] ++ lib.optionals enableCublas [ @@ -246,21 +252,13 @@ let libcufft # cufft.h ]; - cuda-native-redist = symlinkJoin { - name = "cuda-native-redist-${cudaVersion}"; - paths = with cudaPackages; [ - cuda_cudart # cuda_runtime.h - cuda_nvcc - ] ++ cuda-common-redist; - }; - cuda-redist = symlinkJoin { name = "cuda-redist-${cudaVersion}"; paths = cuda-common-redist; }; in -stdenv.mkDerivation { +effectiveStdenv.mkDerivation { pname = "opencv"; inherit version src; @@ -319,7 +317,7 @@ stdenv.mkDerivation { buildInputs = [ zlib pcre boost gflags protobuf3_21 ] ++ lib.optional enablePython pythonPackages.python - ++ lib.optional (stdenv.buildPlatform == stdenv.hostPlatform) hdf5 + ++ lib.optional (effectiveStdenv.buildPlatform == effectiveStdenv.hostPlatform) hdf5 ++ lib.optional enableGtk2 gtk2 ++ lib.optional enableGtk3 gtk3 ++ lib.optional enableVtk vtk @@ -330,7 +328,7 @@ stdenv.mkDerivation { ++ lib.optionals enableEXR [ openexr ilmbase ] ++ lib.optional enableJPEG2000 openjpeg ++ lib.optional enableFfmpeg ffmpeg - ++ lib.optionals (enableFfmpeg && stdenv.isDarwin) + ++ lib.optionals (enableFfmpeg && effectiveStdenv.isDarwin) [ VideoDecodeAcceleration bzip2 ] ++ lib.optionals enableGStreamer (with gst_all_1; [ gstreamer gst-plugins-base gst-plugins-good ]) ++ lib.optional enableOvis ogre @@ -343,7 +341,7 @@ stdenv.mkDerivation { # tesseract & leptonica. ++ lib.optionals enableTesseract [ tesseract leptonica ] ++ lib.optional enableTbb tbb - ++ lib.optionals stdenv.isDarwin [ + ++ lib.optionals effectiveStdenv.isDarwin [ bzip2 AVFoundation Cocoa VideoDecodeAcceleration CoreMedia MediaToolbox Accelerate ] ++ lib.optionals enableDocs [ doxygen graphviz-nox ] @@ -357,7 +355,9 @@ stdenv.mkDerivation { pythonPackages.pip pythonPackages.wheel pythonPackages.setuptools - ] ++ lib.optionals enableCuda [ cuda-native-redist ]; + ] ++ lib.optionals enableCuda [ + cudaPackages.cuda_nvcc + ]; env.NIX_CFLAGS_COMPILE = lib.optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR"; @@ -400,17 +400,14 @@ stdenv.mkDerivation { (opencvFlag "ENABLE_LTO" enableLto) (opencvFlag "ENABLE_THIN_LTO" ( enableLto && ( - # Only clang supports thin LTO, so we must either be using clang through the stdenv, - stdenv.cc.isClang || - # or through the backend stdenv. - (enableCuda && backendStdenv.cc.isClang) + # Only clang supports thin LTO, so we must either be using clang through the effectiveStdenv, + effectiveStdenv.cc.isClang || + # or through the backend effectiveStdenv. + (enableCuda && effectiveStdenv.cc.isClang) ) )) ] ++ lib.optionals enableCuda [ "-DCUDA_FAST_MATH=ON" - # We need to set the C and C++ host compilers for CUDA to the same compiler. - "-DCMAKE_C_COMPILER=${backendStdenv.cc}/bin/cc" - "-DCMAKE_CXX_COMPILER=${backendStdenv.cc}/bin/c++" "-DCUDA_NVCC_FLAGS=--expt-relaxed-constexpr" # OpenCV respects at least three variables: @@ -421,7 +418,7 @@ stdenv.mkDerivation { "-DCUDA_ARCH_PTX=${lib.last cudaCapabilities}" "-DNVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH=${nvidia-optical-flow-sdk}" - ] ++ lib.optionals stdenv.isDarwin [ + ] ++ lib.optionals effectiveStdenv.isDarwin [ "-DWITH_OPENCL=OFF" "-DWITH_LAPACK=OFF" @@ -435,7 +432,7 @@ stdenv.mkDerivation { "-DBUILD_JPEG=OFF" "-DBUILD_PNG=OFF" "-DBUILD_WEBP=OFF" - ] ++ lib.optionals (!stdenv.isDarwin) [ + ] ++ lib.optionals (!effectiveStdenv.isDarwin) [ "-DOPENCL_LIBRARY=${ocl-icd}/lib/libOpenCL.so" ] ++ lib.optionals enablePython [ "-DOPENCV_SKIP_PYTHON_LOADER=ON" @@ -489,9 +486,9 @@ stdenv.mkDerivation { tests = { inherit (gst_all_1) gst-plugins-bad; } - // lib.optionalAttrs (!stdenv.isDarwin) { inherit qimgv; } + // lib.optionalAttrs (!effectiveStdenv.isDarwin) { inherit qimgv; } // lib.optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; } - // lib.optionalAttrs (stdenv.buildPlatform != "x86_64-darwin") { + // lib.optionalAttrs (effectiveStdenv.buildPlatform != "x86_64-darwin") { opencv4-tests = callPackage ./tests.nix { inherit enableGStreamer enableGtk2 enableGtk3 runAccuracyTests runPerformanceTests testDataSrc; inherit opencv4; From c7dbb3ccc9795a85f9bda642f783754deae7c111 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 2 Nov 2023 18:35:26 +0000 Subject: [PATCH 3/5] opencv3: cuda: add a libstdc++ integration test --- pkgs/development/libraries/opencv/3.x.nix | 7 ++++++- pkgs/development/libraries/opencv/4.x.nix | 2 +- pkgs/development/libraries/opencv/libstdcxx-test.nix | 10 +++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pkgs/development/libraries/opencv/3.x.nix b/pkgs/development/libraries/opencv/3.x.nix index 037192d871e7..33bb5bfae26d 100644 --- a/pkgs/development/libraries/opencv/3.x.nix +++ b/pkgs/development/libraries/opencv/3.x.nix @@ -1,6 +1,7 @@ { lib, stdenv , fetchFromGitHub , fetchpatch +, callPackage , cmake, pkg-config, unzip, zlib, pcre, hdf5 , glog, boost, gflags, protobuf3_21 , config @@ -289,7 +290,11 @@ stdenv.mkDerivation { hardeningDisable = [ "bindnow" "relro" ]; - passthru = lib.optionalAttrs enablePython { pythonPath = []; }; + passthru = lib.optionalAttrs enablePython { pythonPath = []; } // { + tests = lib.optionalAttrs enableCuda { + no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { attrName = "opencv3"; }; + }; + }; meta = with lib; { description = "Open Computer Vision Library with more than 500 algorithms"; diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index 87b116f92eb0..21fcfc432d78 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -495,7 +495,7 @@ effectiveStdenv.mkDerivation { }; } // lib.optionalAttrs (enableCuda) { - no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { }; + no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { attrName = "opencv4"; }; }; } // lib.optionalAttrs enablePython { pythonPath = [ ]; }; diff --git a/pkgs/development/libraries/opencv/libstdcxx-test.nix b/pkgs/development/libraries/opencv/libstdcxx-test.nix index aab1101883c2..d8cc483da9a8 100644 --- a/pkgs/development/libraries/opencv/libstdcxx-test.nix +++ b/pkgs/development/libraries/opencv/libstdcxx-test.nix @@ -1,11 +1,11 @@ -{ python3Packages, runCommand }: +{ python3Packages, runCommand, attrName }: -runCommand "${python3Packages.opencv4.pname}-libstdcxx-test" +runCommand "${python3Packages.${attrName}.name}-libstdcxx-test" { nativeBuildInputs = [ - (python3Packages.python.withPackages (ps: with ps; [ - (opencv4.override { enableCuda = true; }) - scikit-image + (python3Packages.python.withPackages (ps: [ + (ps.${attrName}.override { enableCuda = true; }) + ps.scikit-image ])) ]; } '' From 866daf5f7c9d06464b2007070c1c3e4183603f6f Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Thu, 2 Nov 2023 18:18:35 +0000 Subject: [PATCH 4/5] opencv{3,4}: do not ignore config.cudaSupport i686 isn't in the meta.platforms for cudaPackages anyway --- pkgs/development/libraries/opencv/3.x.nix | 3 +-- pkgs/development/libraries/opencv/4.x.nix | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/development/libraries/opencv/3.x.nix b/pkgs/development/libraries/opencv/3.x.nix index 33bb5bfae26d..e65f7ce01da9 100644 --- a/pkgs/development/libraries/opencv/3.x.nix +++ b/pkgs/development/libraries/opencv/3.x.nix @@ -15,8 +15,7 @@ , enableOpenblas ? true, openblas, blas, lapack , enableContrib ? true -, enableCuda ? config.cudaSupport && - stdenv.hostPlatform.isx86_64 +, enableCuda ? config.cudaSupport , cudaPackages ? { } , enableUnfree ? false , enableIpp ? false diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index 21fcfc432d78..05bb780c2524 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -37,7 +37,7 @@ , blas , enableContrib ? true -, enableCuda ? config.cudaSupport && stdenv.hostPlatform.isx86_64 +, enableCuda ? config.cudaSupport , enableCublas ? enableCuda , enableCudnn ? false # NOTE: CUDNN has a large impact on closure size so we disable it by default , enableCufft ? enableCuda From d4229c0e327a09eba11b7472c5c329aa44e961bf Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 3 Nov 2023 13:34:48 +0000 Subject: [PATCH 5/5] opecv4: drop symlinkJoin --- pkgs/development/libraries/opencv/4.x.nix | 29 +++++++++-------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index 05bb780c2524..01e96e8f5b4d 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -42,7 +42,6 @@ , enableCudnn ? false # NOTE: CUDNN has a large impact on closure size so we disable it by default , enableCufft ? enableCuda , cudaPackages ? {} -, symlinkJoin , nvidia-optical-flow-sdk , enableLto ? true @@ -240,22 +239,6 @@ let inherit (cudaPackages) cudaFlags cudaVersion; inherit (cudaFlags) cudaCapabilities; - cuda-common-redist = with cudaPackages; [ - cuda_cudart - cuda_cccl # - libnpp # npp.h - ] ++ lib.optionals enableCublas [ - libcublas # cublas_v2.h - ] ++ lib.optionals enableCudnn [ - cudnn # cudnn.h - ] ++ lib.optionals enableCufft [ - libcufft # cufft.h - ]; - - cuda-redist = symlinkJoin { - name = "cuda-redist-${cudaVersion}"; - paths = cuda-common-redist; - }; in effectiveStdenv.mkDerivation { @@ -345,7 +328,17 @@ effectiveStdenv.mkDerivation { bzip2 AVFoundation Cocoa VideoDecodeAcceleration CoreMedia MediaToolbox Accelerate ] ++ lib.optionals enableDocs [ doxygen graphviz-nox ] - ++ lib.optionals enableCuda [ cuda-redist ]; + ++ lib.optionals enableCuda (with cudaPackages; [ + cuda_cudart + cuda_cccl # + libnpp # npp.h + ] ++ lib.optionals enableCublas [ + libcublas # cublas_v2.h + ] ++ lib.optionals enableCudnn [ + cudnn # cudnn.h + ] ++ lib.optionals enableCufft [ + libcufft # cufft.h + ]); propagatedBuildInputs = lib.optional enablePython pythonPackages.numpy ++ lib.optionals enableCuda [ nvidia-optical-flow-sdk ];