From 8262bdf73850f5ca7f2045a6d82568bb342603a4 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Wed, 3 Apr 2024 09:17:17 +0000 Subject: [PATCH 1/2] cudaPackages.cudatoolkit: replace with symlinkJoin --- .../manual/release-notes/rl-2405.section.md | 6 ++ .../cuda-modules/cudatoolkit/default.nix | 2 +- .../cudatoolkit/redist-wrapper.nix | 86 +++++++++++++++++++ pkgs/top-level/cuda-packages.nix | 6 +- 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/cuda-modules/cudatoolkit/redist-wrapper.nix diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md index a8cefa0da604..540cd7989d01 100644 --- a/nixos/doc/manual/release-notes/rl-2405.section.md +++ b/nixos/doc/manual/release-notes/rl-2405.section.md @@ -307,6 +307,12 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m - The `cudaPackages` package scope has been updated to `cudaPackages_12`. +- The deprecated `cudaPackages.cudatoolkit` has been replaced with a + symlink-based wrapper for the splayed redistributable CUDA packages. The + wrapper only includes tools and libraries necessary to build common packages + like e.g. tensorflow. The original runfile-based `cudatoolkit` is still + available as `cudatoolkit-legacy-runfile`. + - The `halloy` package was updated past 2024.5 which introduced a breaking change by switching the config format from YAML to TOML. See https://github.com/squidowl/halloy/releases/tag/2024.5 for details. - Ada packages (libraries and tools) have been moved into the `gnatPackages` scope. `gnatPackages` uses the default GNAT compiler, `gnat12Packages` and `gnat13Packages` use the respective matching compiler version. diff --git a/pkgs/development/cuda-modules/cudatoolkit/default.nix b/pkgs/development/cuda-modules/cudatoolkit/default.nix index 5a983aaf5c70..e5606f939512 100644 --- a/pkgs/development/cuda-modules/cudatoolkit/default.nix +++ b/pkgs/development/cuda-modules/cudatoolkit/default.nix @@ -411,7 +411,7 @@ backendStdenv.mkDerivation rec { }; meta = with lib; { - description = "A compiler for NVIDIA GPUs, math libraries, and tools"; + description = "The deprecated runfile-based CUDAToolkit installation (a compiler for NVIDIA GPUs, math libraries, and tools)"; homepage = "https://developer.nvidia.com/cuda-toolkit"; platforms = [ "x86_64-linux" ]; license = licenses.nvidiaCuda; diff --git a/pkgs/development/cuda-modules/cudatoolkit/redist-wrapper.nix b/pkgs/development/cuda-modules/cudatoolkit/redist-wrapper.nix new file mode 100644 index 000000000000..6bdcdecbacd6 --- /dev/null +++ b/pkgs/development/cuda-modules/cudatoolkit/redist-wrapper.nix @@ -0,0 +1,86 @@ +{ + lib, + symlinkJoin, + backendStdenv, + cudaOlder, + cudatoolkit-legacy-runfile, + cudaVersion, + cuda_cccl ? null, + cuda_cudart ? null, + cuda_cuobjdump ? null, + cuda_cupti ? null, + cuda_cuxxfilt ? null, + cuda_gdb ? null, + cuda_nvcc ? null, + cuda_nvdisasm ? null, + cuda_nvml_dev ? null, + cuda_nvprune ? null, + cuda_nvrtc ? null, + cuda_nvtx ? null, + cuda_profiler_api ? null, + cuda_sanitizer_api ? null, + libcublas ? null, + libcufft ? null, + libcurand ? null, + libcusolver ? null, + libcusparse ? null, + libnpp ? null, +}: + +let + getAllOutputs = p: [ + (lib.getBin p) + (lib.getLib p) + (lib.getDev p) + ]; + hostPackages = [ + cuda_cuobjdump + cuda_gdb + cuda_nvcc + cuda_nvdisasm + cuda_nvprune + ]; + targetPackages = [ + cuda_cccl + cuda_cudart + cuda_cupti + cuda_cuxxfilt + cuda_nvml_dev + cuda_nvrtc + cuda_nvtx + cuda_profiler_api + cuda_sanitizer_api + libcublas + libcufft + libcurand + libcusolver + libcusparse + libnpp + ]; + + # This assumes we put `cudatoolkit` in `buildInputs` instead of `nativeBuildInputs`: + allPackages = (map (p: p.__spliced.buildHost or p) hostPackages) ++ targetPackages; +in + +if cudaOlder "11.4" then + cudatoolkit-legacy-runfile +else + symlinkJoin rec { + name = "cuda-merged-${cudaVersion}"; + version = cudaVersion; + + paths = builtins.concatMap getAllOutputs allPackages; + + passthru = { + cc = lib.warn "cudaPackages.cudatoolkit is deprecated, refer to the manual and use splayed packages instead" backendStdenv.cc; + lib = symlinkJoin { + inherit name; + paths = map (p: lib.getLib p) allPackages; + }; + }; + + meta = with lib; { + description = "A wrapper substituting the deprecated runfile-based CUDA installation"; + license = licenses.nvidiaCuda; + }; + } diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index f573d2358147..92960422af44 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -69,7 +69,11 @@ let backendStdenv = final.callPackage ../development/cuda-modules/backend-stdenv.nix { }; # Loose packages - cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit { }; + + # TODO: Move to aliases.nix once all Nixpkgs has migrated to the splayed CUDA packages + cudatoolkit = final.callPackage ../development/cuda-modules/cudatoolkit/redist-wrapper.nix { }; + cudatoolkit-legacy-runfile = final.callPackage ../development/cuda-modules/cudatoolkit { }; + saxpy = final.callPackage ../development/cuda-modules/saxpy { }; nccl = final.callPackage ../development/cuda-modules/nccl { }; nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; From 2c51064b0bb9e8ef60d4dab9bf7d26678114c24b Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Sat, 13 Apr 2024 14:24:39 +0000 Subject: [PATCH 2/2] treewide: remove cudatoolkit.cc references --- pkgs/applications/science/math/caffe/default.nix | 4 ++-- pkgs/applications/science/math/mxnet/default.nix | 4 ++-- pkgs/development/libraries/lightgbm/default.nix | 2 +- pkgs/development/libraries/opencv/3.x.nix | 4 ++-- pkgs/development/libraries/xgboost/default.nix | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgs/applications/science/math/caffe/default.nix b/pkgs/applications/science/math/caffe/default.nix index 25f7229a845a..2a84d4b27b14 100644 --- a/pkgs/applications/science/math/caffe/default.nix +++ b/pkgs/applications/science/math/caffe/default.nix @@ -22,7 +22,7 @@ }: let - inherit (cudaPackages) cudatoolkit nccl; + inherit (cudaPackages) backendStdenv cudatoolkit nccl; # The default for cudatoolkit 10.1 is CUDNN 8.0.5, the last version to support CUDA 10.1. # However, this caffe does not build with CUDNN 8.x, so we use CUDNN 7.6.5 instead. # Earlier versions of cudatoolkit use pre-8.x CUDNN, so we use the default. @@ -59,7 +59,7 @@ stdenv.mkDerivation rec { "-DBLAS=open" ] ++ (if cudaSupport then [ "-DCUDA_ARCH_NAME=All" - "-DCUDA_HOST_COMPILER=${cudatoolkit.cc}/bin/cc" + "-DCUDA_HOST_COMPILER=${backendStdenv.cc}/bin/cc" ] else [ "-DCPU_ONLY=ON" ]) ++ ["-DUSE_NCCL=${toggle ncclSupport}"] ++ ["-DUSE_LEVELDB=${toggle leveldbSupport}"] diff --git a/pkgs/applications/science/math/mxnet/default.nix b/pkgs/applications/science/math/mxnet/default.nix index 993da2b89898..839537438962 100644 --- a/pkgs/applications/science/math/mxnet/default.nix +++ b/pkgs/applications/science/math/mxnet/default.nix @@ -5,7 +5,7 @@ }: let - inherit (cudaPackages) cudatoolkit cudaFlags cudnn; + inherit (cudaPackages) backendStdenv cudatoolkit cudaFlags cudnn; in assert cudnnSupport -> cudaSupport; @@ -49,7 +49,7 @@ stdenv.mkDerivation rec { ++ (if cudaSupport then [ "-DUSE_OLDCMAKECUDA=ON" # see https://github.com/apache/incubator-mxnet/issues/10743 "-DCUDA_ARCH_NAME=All" - "-DCUDA_HOST_COMPILER=${cudatoolkit.cc}/bin/cc" + "-DCUDA_HOST_COMPILER=${backendStdenv.cc}/bin/cc" "-DMXNET_CUDA_ARCH=${builtins.concatStringsSep ";" cudaFlags.realArches}" ] else [ "-DUSE_CUDA=OFF" ]) ++ lib.optional (!cudnnSupport) "-DUSE_CUDNN=OFF"; diff --git a/pkgs/development/libraries/lightgbm/default.nix b/pkgs/development/libraries/lightgbm/default.nix index 5f8e6ceaecd0..57d3407aef91 100644 --- a/pkgs/development/libraries/lightgbm/default.nix +++ b/pkgs/development/libraries/lightgbm/default.nix @@ -77,7 +77,7 @@ stdenv.mkDerivation rec { ''; cmakeFlags = lib.optionals doCheck [ "-DBUILD_CPP_TEST=ON" ] - ++ lib.optionals cudaSupport [ "-DUSE_CUDA=1" "-DCMAKE_CXX_COMPILER=${cudaPackages.cudatoolkit.cc}/bin/cc" ] + ++ lib.optionals cudaSupport [ "-DUSE_CUDA=1" "-DCMAKE_CXX_COMPILER=${cudaPackages.backendStdenv.cc}/bin/cc" ] ++ lib.optionals openclSupport [ "-DUSE_GPU=ON" ] ++ lib.optionals mpiSupport [ "-DUSE_MPI=ON" ] ++ lib.optionals hdfsSupport [ diff --git a/pkgs/development/libraries/opencv/3.x.nix b/pkgs/development/libraries/opencv/3.x.nix index 377ebb3acdb5..5e9409368cc7 100644 --- a/pkgs/development/libraries/opencv/3.x.nix +++ b/pkgs/development/libraries/opencv/3.x.nix @@ -39,7 +39,7 @@ assert blas.implementation == "openblas" && lapack.implementation == "openblas"; assert enablePython -> pythonPackages != null; let - inherit (cudaPackages) cudatoolkit; + inherit (cudaPackages) backendStdenv cudatoolkit; inherit (cudaPackages.cudaFlags) cudaCapabilities; version = "3.4.18"; @@ -241,7 +241,7 @@ stdenv.mkDerivation { (opencvFlag "TBB" enableTbb) ] ++ lib.optionals enableCuda [ "-DCUDA_FAST_MATH=ON" - "-DCUDA_HOST_COMPILER=${cudatoolkit.cc}/bin/cc" + "-DCUDA_HOST_COMPILER=${backendStdenv.cc}/bin/cc" "-DCUDA_NVCC_FLAGS=--expt-relaxed-constexpr" "-DCUDA_ARCH_BIN=${lib.concatStringsSep ";" cudaCapabilities}" "-DCUDA_ARCH_PTX=${lib.last cudaCapabilities}" diff --git a/pkgs/development/libraries/xgboost/default.nix b/pkgs/development/libraries/xgboost/default.nix index a1ae57f8824f..0018a73341e3 100644 --- a/pkgs/development/libraries/xgboost/default.nix +++ b/pkgs/development/libraries/xgboost/default.nix @@ -75,8 +75,8 @@ stdenv.mkDerivation rec { "-DUSE_CUDA=ON" # Their CMakeLists.txt does not respect CUDA_HOST_COMPILER, instead using the CXX compiler. # https://github.com/dmlc/xgboost/blob/ccf43d4ba0a94e2f0a3cc5a526197539ae46f410/CMakeLists.txt#L145 - "-DCMAKE_C_COMPILER=${cudaPackages.cudatoolkit.cc}/bin/gcc" - "-DCMAKE_CXX_COMPILER=${cudaPackages.cudatoolkit.cc}/bin/g++" + "-DCMAKE_C_COMPILER=${cudaPackages.backendStdenv.cc}/bin/gcc" + "-DCMAKE_CXX_COMPILER=${cudaPackages.backendStdenv.cc}/bin/g++" ] ++ lib.optionals (cudaSupport && lib.versionAtLeast cudaPackages.cudatoolkit.version "11.4.0")