rocm-related: standardize and fix tests/benchmarks

rocblas: standardize and fix test/benchmarks rocthrust: standardize and fix test/benchmarks rocwmma: standardize and fix tests/benchmarks
2022-12-16 00:51:10 -06:00 · 2022-12-16 00:51:10 -06:00 · fb8a7b62bb
commit fb8a7b62bb
parent 785ed480aa
5 changed files with 98 additions and 80 deletions
--- a/pkgs/development/libraries/rocblas/default.nix
+++ b/pkgs/development/libraries/rocblas/default.nix
@ -2,20 +2,17 @@
 , stdenv
 , fetchFromGitHub
 , rocmUpdateScript
-, fetchpatch
 , cmake
 , rocm-cmake
-, rocm-runtime
-, rocm-device-libs
-, rocm-comgr
 , hip
 , python3
 , tensile
 , msgpack
 , libxml2
-, llvm
 , gtest
 , gfortran
+, openmp
+, amd-blis
 , python3Packages
 , buildTensile ? true
 , buildTests ? false
@ -32,6 +29,14 @@ stdenv.mkDerivation (finalAttrs: {
  pname = "rocblas";
  version = "5.4.0";

+  outputs = [
+    "out"
+  ] ++ lib.optionals buildTests [
+    "test"
+  ] ++ lib.optionals buildBenchmarks [
+    "benchmark"
+  ];
+
  src = fetchFromGitHub {
    owner = "ROCmSoftwarePlatform";
    repo = "rocBLAS";
@ -39,16 +44,6 @@ stdenv.mkDerivation (finalAttrs: {
    hash = "sha256-4art8/KwH2KDLwSYcyzn/m/xwdg5wQQvgHks73aB+60=";
  };

-  # We currently need this patch due to faulty toolchain includes
-  # See: https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1277
-  patches = [
-    (fetchpatch {
-      name = "only-std_norm-from-rocblas_complex.patch";
-      url = "https://github.com/ROCmSoftwarePlatform/rocBLAS/commit/44b99c6df26002139ca9ec68ee1fc8899c7b001f.patch";
-      hash = "sha256-vSZkVYY951fqfOThKFqnYBasWMblS6peEJZ6sFMCk9k=";
-    })
-  ];
-
  nativeBuildInputs = [
    cmake
    rocm-cmake
@ -56,19 +51,19 @@ stdenv.mkDerivation (finalAttrs: {
  ];

  buildInputs = [
-    rocm-runtime
-    rocm-device-libs
-    rocm-comgr
    python3
  ] ++ lib.optionals buildTensile [
    msgpack
    libxml2
-    llvm
-    python3Packages.pyyaml
    python3Packages.msgpack
  ] ++ lib.optionals buildTests [
    gtest
+  ] ++ lib.optionals (buildTests || buildBenchmarks) [
    gfortran
+    openmp
+    amd-blis
+  ] ++ lib.optionals (buildTensile || buildTests || buildBenchmarks) [
+    python3Packages.pyyaml
  ];

  cmakeFlags = [
@ -95,12 +90,12 @@ stdenv.mkDerivation (finalAttrs: {
    "-DBUILD_CLIENTS_TESTS=ON"
  ] ++ lib.optionals buildBenchmarks [
    "-DBUILD_CLIENTS_BENCHMARKS=ON"
+  ] ++ lib.optionals (buildTests || buildBenchmarks) [
+    "-DCMAKE_CXX_FLAGS=-I${amd-blis}/include/blis"
  ];

  # Tensile REALLY wants to write to the nix directory if we include it normally
-  # We need to manually fixup the path so tensile will generate .co and .dat files
  postPatch = lib.optionalString buildTensile ''
-    export PATH=${llvm}/bin:$PATH
    cp -a ${tensile} tensile
    chmod +w -R tensile

@ -113,6 +108,18 @@ stdenv.mkDerivation (finalAttrs: {
      --replace "virtualenv_install(\''${Tensile_TEST_LOCAL_PATH})" ""
  '';

+  postInstall = lib.optionalString buildTests ''
+    mkdir -p $test/bin
+    cp -a $out/bin/* $test/bin
+    rm $test/bin/*-bench || true
+  '' + lib.optionalString buildBenchmarks ''
+    mkdir -p $benchmark/bin
+    cp -a $out/bin/* $benchmark/bin
+    rm $benchmark/bin/*-test || true
+  '' + lib.optionalString (buildTests || buildBenchmarks ) ''
+    rm -rf $out/bin
+  '';
+
  passthru.updateScript = rocmUpdateScript {
    name = finalAttrs.pname;
    owner = finalAttrs.src.owner;
@ -124,8 +131,6 @@ stdenv.mkDerivation (finalAttrs: {
    homepage = "https://github.com/ROCmSoftwarePlatform/rocBLAS";
    license = with licenses; [ mit ];
    maintainers = teams.rocm.members;
-    # Tests and benchmarks are a can of worms that I will tackle in a different PR
-    # It involves completely rewriting the amd-blis derivation
-    broken = finalAttrs.version != hip.version || buildTests || buildBenchmarks;
+    broken = finalAttrs.version != hip.version;
  };
 })
--- a/pkgs/development/libraries/rocthrust/default.nix
+++ b/pkgs/development/libraries/rocthrust/default.nix
@ -4,9 +4,6 @@
 , rocmUpdateScript
 , cmake
 , rocm-cmake
-, rocm-runtime
-, rocm-device-libs
-, rocm-comgr
 , rocprim
 , hip
 , gtest
@ -18,14 +15,13 @@ stdenv.mkDerivation (finalAttrs: {
  pname = "rocthrust";
  version = "5.4.0";

-  # Comment out these outputs until tests/benchmarks are fixed (upstream?)
-  # outputs = [
-  #   "out"
-  # ] ++ lib.optionals buildTests [
-  #   "test"
-  # ] ++ lib.optionals buildBenchmarks [
-  #   "benchmark"
-  # ];
+  outputs = [
+    "out"
+  ] ++ lib.optionals buildTests [
+    "test"
+  ] ++ lib.optionals buildBenchmarks [
+    "benchmark"
+  ];

  src = fetchFromGitHub {
    owner = "ROCmSoftwarePlatform";
@ -41,11 +37,7 @@ stdenv.mkDerivation (finalAttrs: {
    hip
  ];

-  buildInputs = [
-    rocm-runtime
-    rocm-device-libs
-    rocm-comgr
-  ] ++ lib.optionals buildTests [
+  buildInputs = lib.optionals buildTests [
    gtest
  ];

@ -61,18 +53,19 @@ stdenv.mkDerivation (finalAttrs: {
    "-DBUILD_TEST=ON"
  ] ++ lib.optionals buildBenchmarks [
    "-DBUILD_BENCHMARKS=ON"
+  ] ++ lib.optionals (buildTests || buildBenchmarks) [
+    "-DCMAKE_CXX_FLAGS=-Wno-deprecated-builtins" # Too much spam
  ];

-  # Comment out these outputs until tests/benchmarks are fixed (upstream?)
-  # postInstall = lib.optionalString buildTests ''
-  #   mkdir -p $test/bin
-  #   mv $out/bin/test_* $test/bin
-  # '' + lib.optionalString buildBenchmarks ''
-  #   mkdir -p $benchmark/bin
-  #   mv $out/bin/benchmark_* $benchmark/bin
-  # '' + lib.optionalString (buildTests || buildBenchmarks) ''
-  #   rmdir $out/bin
-  # '';
+  postInstall = lib.optionalString buildTests ''
+    mkdir -p $test/bin
+    mv $out/bin/{test_*,*.hip} $test/bin
+  '' + lib.optionalString buildBenchmarks ''
+    mkdir -p $benchmark/bin
+    mv $out/bin/benchmark_* $benchmark/bin
+  '' + lib.optionalString (buildTests || buildBenchmarks) ''
+    rm -rf $out/bin
+  '';

  passthru.updateScript = rocmUpdateScript {
    name = finalAttrs.pname;
@ -85,8 +78,6 @@ stdenv.mkDerivation (finalAttrs: {
    homepage = "https://github.com/ROCmSoftwarePlatform/rocThrust";
    license = with licenses; [ asl20 ];
    maintainers = teams.rocm.members;
-    # Tests/Benchmarks don't seem to work, thousands of errors compiling with no clear fix
-    # Is this an upstream issue? We don't seem to be missing dependencies
-    broken = finalAttrs.version != hip.version || buildTests || buildBenchmarks;
+    broken = finalAttrs.version != hip.version;
  };
 })
--- a/pkgs/development/libraries/rocwmma/0000-dont-fetch-googletest.patch
+++ b/pkgs/development/libraries/rocwmma/0000-dont-fetch-googletest.patch
@ -1,19 +1,31 @@
 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
-index 7e0f2c8..db54eab 100644
+index e1160bb..2a5462e 100644
 --- a/test/CMakeLists.txt
 +++ b/test/CMakeLists.txt
-@@ -31,14 +31,6 @@ cmake_dependent_option( ROCWMMA_BUILD_BENCHMARK_TESTS "Build benchmarking tests"
+@@ -30,26 +30,6 @@ cmake_dependent_option( ROCWMMA_BUILD_VALIDATION_TESTS "Build validation tests"
+ cmake_dependent_option( ROCWMMA_BUILD_BENCHMARK_TESTS "Build benchmarking tests" OFF "ROCWMMA_BUILD_TESTS" OFF )
 cmake_dependent_option( ROCWMMA_BUILD_EXTENDED_TESTS "Build extended test parameter coverage" OFF "ROCWMMA_BUILD_TESTS" OFF )
 
- # Test/benchmark requires additional dependencies
+-# Test/benchmark requires additional dependencies
 -include( FetchContent )
 -
 -FetchContent_Declare(
 -  googletest
-  URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
+-  GIT_REPOSITORY https://github.com/google/googletest.git
+-  GIT_TAG 609281088cfefc76f9d0ce82e1ff6c30cc3591e5
 -)
-FetchContent_MakeAvailable(googletest)
-
- include(GoogleTest)
+-FetchContent_GetProperties(googletest)
+-if(NOT googletest_POPULATED)
+-  # Fetch the content using default details
+-  FetchContent_Populate(googletest)
+-  # Save the shared libs setting, then force to static libs
+-  set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
+-  set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "Build SHARED libraries" FORCE)
+-  # Add gtest targets as static libs
+-  add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR})
+-  # Restore shared libs setting
+-  set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_OLD} CACHE INTERNAL "Build SHARED libraries" FORCE)
+-endif()
 
- set(ROCWMMA_TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+ set(ROCWMMA_TEST_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR})
+ set(ROCWMMA_COMMON_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/hip_device.cpp
--- a/pkgs/development/libraries/rocwmma/default.nix
+++ b/pkgs/development/libraries/rocwmma/default.nix
@ -14,6 +14,8 @@
 , python3Packages
 , buildDocs ? true
 , buildTests ? false
+, buildExtendedTests ? false
+, buildBenchmarks ? false
 , buildSamples ? false
 , gpuTargets ? [ ] # gpuTargets = [ "gfx908:xnack-" "gfx90a:xnack-" "gfx90a:xnack+" ... ]
 }:
@ -40,8 +42,10 @@ in stdenv.mkDerivation (finalAttrs: {
    "out"
  ] ++ lib.optionals buildDocs [
    "doc"
-  ] ++ lib.optionals buildTests [
+  ] ++ lib.optionals (buildTests || buildBenchmarks) [
    "test"
+  ] ++ lib.optionals buildBenchmarks [
+    "benchmark"
  ] ++ lib.optionals buildSamples [
    "sample"
  ];
@ -53,7 +57,7 @@ in stdenv.mkDerivation (finalAttrs: {
    hash = "sha256-HUJPb6IahBgl/v+W4kXludBTNAjRm8k6v0jxKAX+qZM=";
  };

-  patches = lib.optionals buildTests [
+  patches = lib.optionals (buildTests || buildBenchmarks) [
    ./0000-dont-fetch-googletest.patch
  ];

@ -65,7 +69,7 @@ in stdenv.mkDerivation (finalAttrs: {

  buildInputs = [
    openmp
-  ] ++ lib.optionals buildTests [
+  ] ++ lib.optionals (buildTests || buildBenchmarks) [
    gtest
    rocblas
  ] ++ lib.optionals buildDocs [
@ -78,8 +82,10 @@ in stdenv.mkDerivation (finalAttrs: {

  cmakeFlags = [
    "-DCMAKE_CXX_COMPILER=hipcc"
-    "-DROCWMMA_BUILD_TESTS=${if buildTests then "ON" else "OFF"}"
+    "-DROCWMMA_BUILD_TESTS=${if buildTests || buildBenchmarks then "ON" else "OFF"}"
+    "-DROCWMMA_BUILD_VALIDATION_TESTS=ON"
    "-DROCWMMA_BUILD_SAMPLES=${if buildSamples then "ON" else "OFF"}"
+    "-DROCWMMA_VALIDATE_WITH_ROCBLAS=ON"
    # Manually define CMAKE_INSTALL_<DIR>
    # See: https://github.com/NixOS/nixpkgs/pull/197838
    "-DCMAKE_INSTALL_BINDIR=bin"
@ -87,11 +93,10 @@ in stdenv.mkDerivation (finalAttrs: {
    "-DCMAKE_INSTALL_INCLUDEDIR=include"
  ] ++ lib.optionals (gpuTargets != [ ]) [
    "-DGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}"
-  ] ++ lib.optionals buildTests [
-    "-DROCWMMA_BUILD_VALIDATION_TESTS=ON"
-    "-DROCWMMA_BUILD_BENCHMARK_TESTS=ON"
+  ] ++ lib.optionals buildExtendedTests [
    "-DROCWMMA_BUILD_EXTENDED_TESTS=ON"
-    "-DROCWMMA_VALIDATE_WITH_ROCBLAS=ON"
+  ] ++ lib.optionals buildBenchmarks [
+    "-DROCWMMA_BUILD_BENCHMARK_TESTS=ON"
    "-DROCWMMA_BENCHMARK_WITH_ROCBLAS=ON"
  ];

@ -109,16 +114,19 @@ in stdenv.mkDerivation (finalAttrs: {
  postInstall = lib.optionalString buildDocs ''
    mv ../docs/source/_build/html $out/share/doc/rocwmma
    mv ../docs/source/_build/latex/rocWMMA.pdf $out/share/doc/rocwmma
-  '' + lib.optionalString buildTests ''
+  '' + lib.optionalString (buildTests || buildBenchmarks) ''
    mkdir -p $test/bin
-    mv $out/bin/*_test* $test/bin
+    mv $out/bin/{*_test,*-validate} $test/bin
+  '' + lib.optionalString buildBenchmarks ''
+    mkdir -p $benchmark/bin
+    mv $out/bin/*-bench $benchmark/bin
  '' + lib.optionalString buildSamples ''
    mkdir -p $sample/bin
    mv $out/bin/sgemmv $sample/bin
    mv $out/bin/simple_gemm $sample/bin
    mv $out/bin/simple_dlrm $sample/bin
-  '' + lib.optionalString (buildTests || buildSamples) ''
-    rmdir $out/bin
+  '' + lib.optionalString (buildTests || buildBenchmarks || buildSamples) ''
+    rm -rf $out/bin
  '';

  passthru.updateScript = rocmUpdateScript {
@ -132,8 +140,6 @@ in stdenv.mkDerivation (finalAttrs: {
    homepage = "https://github.com/ROCmSoftwarePlatform/rocWMMA";
    license = with licenses; [ mit ];
    maintainers = teams.rocm.members;
-    # Building tests isn't working for now
-    # undefined reference to symbol '_ZTIN7testing4TestE'
-    broken = finalAttrs.version != hip.version || buildTests;
+    broken = finalAttrs.version != hip.version;
  };
 })
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -15229,18 +15229,22 @@ with pkgs;
  };

  rocwmma = callPackage ../development/libraries/rocwmma {
-    inherit (llvmPackages) openmp;
+    inherit (llvmPackages_rocm) openmp;
+    stdenv = rocmClangStdenv;
  };

  rocblas = callPackage ../development/libraries/rocblas {
-    inherit (llvmPackages_rocm) llvm;
+    inherit (llvmPackages_rocm) openmp;
+    stdenv = rocmClangStdenv;
  };

  miopengemm = callPackage ../development/libraries/miopengemm {
    stdenv = rocmClangStdenv;
  };

-  rocthrust = callPackage ../development/libraries/rocthrust { };
+  rocthrust = callPackage ../development/libraries/rocthrust {
+    stdenv = rocmClangStdenv;
+  };

  miopen = callPackage ../development/libraries/miopen {
    inherit (llvmPackages_rocm) llvm clang-tools-extra;