From aa117cc20920e3a9591b4b37df1f2b7d0d2d0f8d Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 4 Sep 2024 21:27:05 +0000 Subject: [PATCH 1/2] opencv: inherit from lib, switch to cmake* helpers, fix CUDA --- pkgs/development/libraries/opencv/4.x.nix | 253 ++++++++++++---------- pkgs/top-level/all-packages.nix | 2 + 2 files changed, 138 insertions(+), 117 deletions(-) diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index ef0266154bd5..e979c9aa3863 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -46,7 +46,7 @@ , cudaPackages ? {} , nvidia-optical-flow-sdk -, enableLto ? true +, enableLto ? false # TODO: Investigate LTO build failures , enableUnfree ? false , enableIpp ? false , enablePython ? false @@ -97,6 +97,12 @@ }@inputs: let + inherit (lib.attrsets) mapAttrsToList optionalAttrs; + inherit (lib.lists) last optionals; + inherit (lib.meta) getExe; + inherit (lib.strings) cmakeBool cmakeFeature cmakeOptionType concatStrings concatStringsSep optionalString; + inherit (lib.trivial) flip; + version = "4.9.0"; # It's necessary to consistently use backendStdenv when building with CUDA @@ -228,26 +234,23 @@ let }; # See opencv/cmake/OpenCVDownload.cmake - installExtraFiles = extra: '' - mkdir -p "${extra.dst}" - '' + lib.concatStrings (lib.flip lib.mapAttrsToList extra.files (name: md5: '' - ln -s "${extra.src}/${name}" "${extra.dst}/${md5}-${name}" + installExtraFiles = {dst, files, src, ...}: '' + mkdir -p "${dst}" + '' + concatStrings (flip mapAttrsToList files (name: md5: '' + ln -s "${src}/${name}" "${dst}/${md5}-${name}" '')); - installExtraFile = extra: '' - mkdir -p "${extra.dst}" - ln -s "${extra.src}" "${extra.dst}/${extra.md5}-${extra.name}" + installExtraFile = {dst, md5, name, src, ...}: '' + mkdir -p "${dst}" + ln -s "${src}" "${dst}/${md5}-${name}" ''; - opencvFlag = name: enabled: "-DWITH_${name}=${printEnabled enabled}"; - - printEnabled = enabled: if enabled then "ON" else "OFF"; withOpenblas = (enableBlas && blas.provider.pname == "openblas"); #multithreaded openblas conflicts with opencv multithreading, which manifest itself in hung tests #https://github.com/OpenMathLib/OpenBLAS/wiki/Faq/4bded95e8dc8aadc70ce65267d1093ca7bdefc4c#multi-threaded openblas_ = blas.provider.override { singleThreaded = true; }; - inherit (cudaPackages) cudaFlags cudaVersion; - inherit (cudaFlags) cudaCapabilities; + inherit (cudaPackages) cudaFlags; + inherit (cudaFlags) cmakeCudaArchitecturesString cudaCapabilities; in @@ -258,20 +261,21 @@ effectiveStdenv.mkDerivation { outputs = [ "out" "cxxdev" - ] ++ lib.optionals (runAccuracyTests || runPerformanceTests) [ + ] ++ optionals (runAccuracyTests || runPerformanceTests) [ "package_tests" ]; cudaPropagateToOutput = "cxxdev"; - postUnpack = lib.optionalString buildContrib '' + postUnpack = optionalString buildContrib '' cp --no-preserve=mode -r "${contribSrc}/modules" "$NIX_BUILD_TOP/source/opencv_contrib" ''; # Ensures that we use the system OpenEXR rather than the vendored copy of the source included with OpenCV. patches = [ ./cmake-don-t-use-OpenCVFindOpenEXR.patch - ] ++ lib.optionals enableContrib [ - ] ++ lib.optional enableCuda ./cuda_opt_flow.patch; + ] ++ optionals enableCuda [ + ./cuda_opt_flow.patch + ]; # This prevents cmake from using libraries in impure paths (which # causes build failure on non NixOS) @@ -281,8 +285,8 @@ effectiveStdenv.mkDerivation { preConfigure = installExtraFile ade + - lib.optionalString enableIpp (installExtraFiles ippicv) + ( - lib.optionalString buildContrib '' + optionalString enableIpp (installExtraFiles ippicv) + ( + optionalString buildContrib '' cmakeFlagsArray+=("-DOPENCV_EXTRA_MODULES_PATH=$NIX_BUILD_TOP/source/opencv_contrib") ${installExtraFiles vgg} @@ -304,35 +308,35 @@ effectiveStdenv.mkDerivation { pcre2 protobuf_21 zlib - ] ++ lib.optionals enablePython [ + ] ++ optionals enablePython [ pythonPackages.python - ] ++ lib.optionals (effectiveStdenv.buildPlatform == effectiveStdenv.hostPlatform) [ + ] ++ optionals (effectiveStdenv.buildPlatform == effectiveStdenv.hostPlatform) [ hdf5 - ] ++ lib.optionals enableGtk2 [ + ] ++ optionals enableGtk2 [ gtk2 - ] ++ lib.optionals enableGtk3 [ + ] ++ optionals enableGtk3 [ gtk3 - ] ++ lib.optionals enableVtk [ + ] ++ optionals enableVtk [ vtk - ] ++ lib.optionals enableJPEG [ + ] ++ optionals enableJPEG [ libjpeg - ] ++ lib.optionals enablePNG [ + ] ++ optionals enablePNG [ libpng - ] ++ lib.optionals enableTIFF [ + ] ++ optionals enableTIFF [ libtiff - ] ++ lib.optionals enableWebP [ + ] ++ optionals enableWebP [ libwebp - ] ++ lib.optionals enableEXR [ + ] ++ optionals enableEXR [ openexr ilmbase - ] ++ lib.optionals enableJPEG2000 [ + ] ++ optionals enableJPEG2000 [ openjpeg - ] ++ lib.optionals enableFfmpeg [ + ] ++ optionals enableFfmpeg [ ffmpeg - ] ++ lib.optionals (enableFfmpeg && effectiveStdenv.hostPlatform.isDarwin) [ + ] ++ optionals (enableFfmpeg && effectiveStdenv.hostPlatform.isDarwin) [ bzip2 VideoDecodeAcceleration - ] ++ lib.optionals (enableGStreamer && effectiveStdenv.hostPlatform.isLinux) [ + ] ++ optionals (enableGStreamer && effectiveStdenv.hostPlatform.isLinux) [ elfutils gst_all_1.gst-plugins-base gst_all_1.gst-plugins-good @@ -340,27 +344,27 @@ effectiveStdenv.mkDerivation { libunwind orc zstd - ] ++ lib.optionals enableOvis [ + ] ++ optionals enableOvis [ ogre - ] ++ lib.optionals enableGPhoto2 [ + ] ++ optionals enableGPhoto2 [ libgphoto2 - ] ++ lib.optionals enableDC1394 [ + ] ++ optionals enableDC1394 [ libdc1394 - ] ++ lib.optionals enableEigen [ + ] ++ optionals enableEigen [ eigen - ] ++ lib.optionals enableVA [ + ] ++ optionals enableVA [ libva - ] ++ lib.optionals enableBlas [ + ] ++ optionals enableBlas [ blas.provider - ] ++ lib.optionals enableTesseract [ + ] ++ optionals enableTesseract [ # There is seemingly no compile-time flag for Tesseract. It's # simply enabled automatically if contrib is built, and it detects # tesseract & leptonica. tesseract leptonica - ] ++ lib.optionals enableTbb [ + ] ++ optionals enableTbb [ tbb - ] ++ lib.optionals effectiveStdenv.hostPlatform.isDarwin [ + ] ++ optionals effectiveStdenv.hostPlatform.isDarwin [ bzip2 AVFoundation Cocoa @@ -368,76 +372,78 @@ effectiveStdenv.mkDerivation { CoreMedia MediaToolbox Accelerate - ] ++ lib.optionals enableDocs [ + ] ++ optionals enableDocs [ doxygen graphviz-nox - ] ++ lib.optionals enableCuda [ + ] ++ optionals enableCuda [ cudaPackages.cuda_cudart cudaPackages.cuda_cccl # cudaPackages.libnpp # npp.h nvidia-optical-flow-sdk - ] ++ lib.optionals enableCublas [ + ] ++ optionals enableCublas [ # May start using the default $out instead once # https://github.com/NixOS/nixpkgs/issues/271792 # has been addressed cudaPackages.libcublas # cublas_v2.h - ] ++ lib.optionals enableCudnn [ + ] ++ optionals enableCudnn [ cudaPackages.cudnn # cudnn.h - ] ++ lib.optionals enableCufft [ + ] ++ optionals enableCufft [ cudaPackages.libcufft # cufft.h ]; - propagatedBuildInputs = lib.optionals enablePython [ pythonPackages.numpy ]; + propagatedBuildInputs = optionals enablePython [ pythonPackages.numpy ]; - nativeBuildInputs = [ cmake pkg-config unzip ] - ++ lib.optionals enablePython [ + nativeBuildInputs = [ + cmake + pkg-config + unzip + ] ++ optionals enablePython [ pythonPackages.pip pythonPackages.wheel pythonPackages.setuptools - ] ++ lib.optionals enableCuda [ + ] ++ optionals enableCuda [ cudaPackages.cuda_nvcc ]; - env.NIX_CFLAGS_COMPILE = lib.optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR"; + env.NIX_CFLAGS_COMPILE = optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR"; # Configure can't find the library without this. - OpenBLAS_HOME = lib.optionalString withOpenblas openblas_.dev; - OpenBLAS = lib.optionalString withOpenblas openblas_; + OpenBLAS_HOME = optionalString withOpenblas openblas_.dev; + OpenBLAS = optionalString withOpenblas openblas_; cmakeFlags = [ - "-DOPENCV_GENERATE_PKGCONFIG=ON" - "-DWITH_OPENMP=ON" - "-DBUILD_PROTOBUF=OFF" - "-DProtobuf_PROTOC_EXECUTABLE=${lib.getExe buildPackages.protobuf_21}" - "-DPROTOBUF_UPDATE_FILES=ON" - "-DOPENCV_ENABLE_NONFREE=${printEnabled enableUnfree}" - "-DBUILD_TESTS=${printEnabled runAccuracyTests}" - "-DBUILD_PERF_TESTS=${printEnabled runPerformanceTests}" - "-DCMAKE_SKIP_BUILD_RPATH=ON" - "-DBUILD_DOCS=${printEnabled enableDocs}" + (cmakeBool "OPENCV_GENERATE_PKGCONFIG" true) + (cmakeBool "WITH_OPENMP" true) + (cmakeBool "BUILD_PROTOBUF" false) + (cmakeOptionType "path" "Protobuf_PROTOC_EXECUTABLE" (getExe buildPackages.protobuf_21)) + (cmakeBool "PROTOBUF_UPDATE_FILES" true) + (cmakeBool "OPENCV_ENABLE_NONFREE" enableUnfree) + (cmakeBool "BUILD_TESTS" runAccuracyTests) + (cmakeBool "BUILD_PERF_TESTS" runPerformanceTests) + (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true) + (cmakeBool "BUILD_DOCS" enableDocs) # "OpenCV disables pkg-config to avoid using of host libraries. Consider using PKG_CONFIG_LIBDIR to specify target SYSROOT" # but we have proper separation of build and host libs :), fixes cross - "-DOPENCV_ENABLE_PKG_CONFIG=ON" - (opencvFlag "IPP" enableIpp) - (opencvFlag "TIFF" enableTIFF) - (opencvFlag "WEBP" enableWebP) - (opencvFlag "JPEG" enableJPEG) - (opencvFlag "PNG" enablePNG) - (opencvFlag "OPENEXR" enableEXR) - (opencvFlag "OPENJPEG" enableJPEG2000) - "-DWITH_JASPER=OFF" # OpenCV falls back to a vendored copy of Jasper when OpenJPEG is disabled - (opencvFlag "TBB" enableTbb) + (cmakeBool "OPENCV_ENABLE_PKG_CONFIG" true) + (cmakeBool "WITH_IPP" enableIpp) + (cmakeBool "WITH_TIFF" enableTIFF) + (cmakeBool "WITH_WEBP" enableWebP) + (cmakeBool "WITH_JPEG" enableJPEG) + (cmakeBool "WITH_PNG" enablePNG) + (cmakeBool "WITH_OPENEXR" enableEXR) + (cmakeBool "WITH_OPENJPEG" enableJPEG2000) + (cmakeBool "WITH_JASPER" false) # OpenCV falls back to a vendored copy of Jasper when OpenJPEG is disabled + (cmakeBool "WITH_TBB" enableTbb) # CUDA options - (opencvFlag "CUDA" enableCuda) - (opencvFlag "CUDA_FAST_MATH" enableCuda) - (opencvFlag "CUBLAS" enableCublas) - (opencvFlag "CUDNN" enableCudnn) - (opencvFlag "CUFFT" enableCufft) + (cmakeBool "WITH_CUDA" enableCuda) + (cmakeBool "WITH_CUBLAS" enableCublas) + (cmakeBool "WITH_CUDNN" enableCudnn) + (cmakeBool "WITH_CUFFT" enableCufft) # LTO options - (opencvFlag "ENABLE_LTO" enableLto) - (opencvFlag "ENABLE_THIN_LTO" ( + (cmakeBool "ENABLE_LTO" enableLto) + (cmakeBool "ENABLE_THIN_LTO" ( enableLto && ( # Only clang supports thin LTO, so we must either be using clang through the effectiveStdenv, effectiveStdenv.cc.isClang || @@ -445,51 +451,53 @@ effectiveStdenv.mkDerivation { (enableCuda && effectiveStdenv.cc.isClang) ) )) - ] ++ lib.optionals enableCuda [ - "-DCUDA_FAST_MATH=ON" - "-DCUDA_NVCC_FLAGS=--expt-relaxed-constexpr" + ] ++ optionals enableCuda [ + (cmakeBool "CUDA_FAST_MATH" true) + (cmakeFeature "CUDA_NVCC_FLAGS" "--expt-relaxed-constexpr") # OpenCV respects at least three variables: # -DCUDA_GENERATION takes a single arch name, e.g. Volta # -DCUDA_ARCH_BIN takes a semi-colon separated list of real arches, e.g. "8.0;8.6" # -DCUDA_ARCH_PTX takes the virtual arch, e.g. "8.6" - "-DCUDA_ARCH_BIN=${lib.concatStringsSep ";" cudaCapabilities}" - "-DCUDA_ARCH_PTX=${lib.last cudaCapabilities}" + (cmakeFeature "CUDA_ARCH_BIN" cmakeCudaArchitecturesString) + (cmakeFeature "CUDA_ARCH_PTX" (last cudaCapabilities)) - "-DNVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH=${nvidia-optical-flow-sdk}" - ] ++ lib.optionals effectiveStdenv.hostPlatform.isDarwin [ - "-DWITH_OPENCL=OFF" - "-DWITH_LAPACK=OFF" + (cmakeOptionType "path" "NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH" nvidia-optical-flow-sdk.outPath) + ] ++ optionals effectiveStdenv.hostPlatform.isDarwin [ + (cmakeBool "WITH_OPENCL" false) + (cmakeBool "WITH_LAPACK" false) # Disable unnecessary vendoring that's enabled by default only for Darwin. # Note that the opencvFlag feature flags listed above still take # precedence, so we can safely list everything here. - "-DBUILD_ZLIB=OFF" - "-DBUILD_TIFF=OFF" - "-DBUILD_OPENJPEG=OFF" - "-DBUILD_JASPER=OFF" - "-DBUILD_JPEG=OFF" - "-DBUILD_PNG=OFF" - "-DBUILD_WEBP=OFF" - ] ++ lib.optionals (!effectiveStdenv.hostPlatform.isDarwin) [ - "-DOPENCL_LIBRARY=${ocl-icd}/lib/libOpenCL.so" - ] ++ lib.optionals enablePython [ - "-DOPENCV_SKIP_PYTHON_LOADER=ON" - ] ++ lib.optionals (enabledModules != [ ]) [ - "-DBUILD_LIST=${lib.concatStringsSep "," enabledModules}" + (cmakeBool "BUILD_ZLIB" false) + (cmakeBool "BUILD_TIFF" false) + (cmakeBool "BUILD_OPENJPEG" false) + (cmakeBool "BUILD_JASPER" false) + (cmakeBool "BUILD_JPEG" false) + (cmakeBool "BUILD_PNG" false) + (cmakeBool "BUILD_WEBP" false) + ] ++ optionals (!effectiveStdenv.hostPlatform.isDarwin) [ + (cmakeOptionType "path" "OPENCL_LIBRARY" "${ocl-icd}/lib/libOpenCL.so") + ] ++ optionals enablePython [ + (cmakeBool "OPENCV_SKIP_PYTHON_LOADER" true) + ] ++ optionals (enabledModules != [ ]) [ + (cmakeFeature "BUILD_LIST" (concatStringsSep "," enabledModules)) ]; - postBuild = lib.optionalString enableDocs '' + postBuild = optionalString enableDocs '' make doxygen ''; preInstall = - lib.optionalString (runAccuracyTests || runPerformanceTests) '' - mkdir $package_tests - cp -R $src/samples $package_tests/ - '' - + lib.optionalString runAccuracyTests "mv ./bin/*test* $package_tests/ \n" - + lib.optionalString runPerformanceTests "mv ./bin/*perf* $package_tests/"; + optionalString (runAccuracyTests || runPerformanceTests) '' + mkdir $package_tests + cp -R $src/samples $package_tests/ + '' + optionalString runAccuracyTests '' + mv ./bin/*test* $package_tests/ + '' + optionalString runPerformanceTests '' + mv ./bin/*perf* $package_tests/ + ''; # By default $out/lib/pkgconfig/opencv4.pc looks something like this: # @@ -510,12 +518,23 @@ effectiveStdenv.mkDerivation { '' # fix deps not progagating from opencv4.cxxdev if cuda is disabled # see https://github.com/NixOS/nixpkgs/issues/276691 - + lib.optionalString (!enableCuda) '' + + optionalString (!enableCuda) '' mkdir -p "$cxxdev/nix-support" echo "''${!outputDev}" >> "$cxxdev/nix-support/propagated-build-inputs" '' + # remove the requirement that the exact same version of CUDA is used in packages + # consuming OpenCV's CMakes files + + optionalString enableCuda '' + substituteInPlace "$out/lib/cmake/opencv4/OpenCVConfig.cmake" \ + --replace-fail \ + 'find_host_package(CUDA ''${OpenCV_CUDA_VERSION} EXACT REQUIRED)' \ + 'find_host_package(CUDA REQUIRED)' \ + --replace-fail \ + 'message(FATAL_ERROR "OpenCV static library was compiled with CUDA' \ + 'message("OpenCV static library was compiled with CUDA' + '' # install python distribution information, so other packages can `import opencv` - + lib.optionalString enablePython '' + + optionalString enablePython '' pushd $NIX_BUILD_TOP/$sourceRoot/modules/python/package python -m pip wheel --verbose --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist . @@ -536,18 +555,18 @@ effectiveStdenv.mkDerivation { tests = { inherit (gst_all_1) gst-plugins-bad; } - // lib.optionalAttrs (!effectiveStdenv.hostPlatform.isDarwin) { inherit qimgv; } - // lib.optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; } - // lib.optionalAttrs (effectiveStdenv.buildPlatform != "x86_64-darwin") { + // optionalAttrs (!effectiveStdenv.hostPlatform.isDarwin) { inherit qimgv; } + // optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; } + // optionalAttrs (effectiveStdenv.buildPlatform != "x86_64-darwin") { opencv4-tests = callPackage ./tests.nix { inherit enableGStreamer enableGtk2 enableGtk3 runAccuracyTests runPerformanceTests testDataSrc; inherit opencv4; }; } - // lib.optionalAttrs (enableCuda) { + // optionalAttrs (enableCuda) { no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { attrName = "opencv4"; }; }; - } // lib.optionalAttrs enablePython { pythonPath = [ ]; }; + } // optionalAttrs enablePython { pythonPath = [ ]; }; meta = { description = "Open Computer Vision Library with more than 500 algorithms"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7afe4acb5f0c..837c1326af1f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -22592,6 +22592,8 @@ with pkgs; inherit (darwin.apple_sdk.frameworks) AVFoundation Cocoa VideoDecodeAcceleration CoreMedia MediaToolbox Accelerate; pythonPackages = python3Packages; + # TODO(@connorbaker): OpenCV 4.9 only supports up to CUDA 12.3. + cudaPackages = cudaPackages_12_3; }; opencv4WithoutCuda = opencv4.override { From 7edb29bd29dc2d563fa2269671e2bd1e3f421583 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Mon, 16 Sep 2024 23:18:03 +0000 Subject: [PATCH 2/2] cudaPackages.tests: create tests for OpenCV/Torch ordering --- .../tests/opencv-and-torch/default.nix | 81 +++++++++++++++++++ pkgs/development/libraries/opencv/4.x.nix | 2 +- pkgs/top-level/all-packages.nix | 3 + pkgs/top-level/cuda-packages.nix | 41 ++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 pkgs/development/cuda-modules/tests/opencv-and-torch/default.nix diff --git a/pkgs/development/cuda-modules/tests/opencv-and-torch/default.nix b/pkgs/development/cuda-modules/tests/opencv-and-torch/default.nix new file mode 100644 index 000000000000..442bfb8a2dd0 --- /dev/null +++ b/pkgs/development/cuda-modules/tests/opencv-and-torch/default.nix @@ -0,0 +1,81 @@ +{ + cudaPackages, + lib, + writeGpuTestPython, + # Configuration flags + openCVFirst, + useOpenCVDefaultCuda, + useTorchDefaultCuda, +}: +let + inherit (lib.strings) optionalString; + + openCVBlock = '' + + import cv2 + print("OpenCV version:", cv2.__version__) + + # Ensure OpenCV can access the GPU. + assert cv2.cuda.getCudaEnabledDeviceCount() > 0, "No CUDA devices found for OpenCV" + print("OpenCV CUDA device:", cv2.cuda.printCudaDeviceInfo(cv2.cuda.getDevice())) + + # Ensure OpenCV can access the GPU. + print(cv2.getBuildInformation()) + + a = cv2.cuda.GpuMat(size=(256, 256), type=cv2.CV_32S, s=1) + b = cv2.cuda.GpuMat(size=(256, 256), type=cv2.CV_32S, s=1) + c = int(cv2.cuda.sum(cv2.cuda.add(a, b))[0]) # OpenCV returns a Scalar float object. + + assert c == 2 * 256 * 256, f"Expected {2 * 256 * 256} OpenCV, got {c}" + + ''; + + torchBlock = '' + + import torch + print("Torch version:", torch.__version__) + + # Set up the GPU. + torch.cuda.init() + # Ensure the GPU is available. + assert torch.cuda.is_available(), "CUDA is not available to Torch" + print("Torch CUDA device:", torch.cuda.get_device_properties(torch.cuda.current_device())) + + a = torch.ones(256, 256, dtype=torch.int32).cuda() + b = torch.ones(256, 256, dtype=torch.int32).cuda() + c = (a + b).sum().item() + assert c == 2 * 256 * 256, f"Expected {2 * 256 * 256} for Torch, got {c}" + + ''; + + content = if openCVFirst then openCVBlock + torchBlock else torchBlock + openCVBlock; + + torchName = "torch" + optionalString useTorchDefaultCuda "-with-default-cuda"; + openCVName = "opencv4" + optionalString useOpenCVDefaultCuda "-with-default-cuda"; +in +# TODO: Ensure the expected CUDA libraries are loaded. +# TODO: Ensure GPU access works as expected. +writeGpuTestPython { + name = if openCVFirst then "${openCVName}-then-${torchName}" else "${torchName}-then-${openCVName}"; + libraries = + # NOTE: These are purposefully in this order. + pythonPackages: + let + effectiveOpenCV = pythonPackages.opencv4.override (prevAttrs: { + cudaPackages = if useOpenCVDefaultCuda then prevAttrs.cudaPackages else cudaPackages; + }); + effectiveTorch = pythonPackages.torchWithCuda.override (prevAttrs: { + cudaPackages = if useTorchDefaultCuda then prevAttrs.cudaPackages else cudaPackages; + }); + in + if openCVFirst then + [ + effectiveOpenCV + effectiveTorch + ] + else + [ + effectiveTorch + effectiveOpenCV + ]; +} content diff --git a/pkgs/development/libraries/opencv/4.x.nix b/pkgs/development/libraries/opencv/4.x.nix index e979c9aa3863..3d5c0ac87067 100644 --- a/pkgs/development/libraries/opencv/4.x.nix +++ b/pkgs/development/libraries/opencv/4.x.nix @@ -46,7 +46,7 @@ , cudaPackages ? {} , nvidia-optical-flow-sdk -, enableLto ? false # TODO: Investigate LTO build failures +, enableLto ? true , enableUnfree ? false , enableIpp ? false , enablePython ? false diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 837c1326af1f..8c810af244d0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -22594,6 +22594,9 @@ with pkgs; pythonPackages = python3Packages; # TODO(@connorbaker): OpenCV 4.9 only supports up to CUDA 12.3. cudaPackages = cudaPackages_12_3; + # TODO: LTO does not work. + # https://github.com/NixOS/nixpkgs/issues/343123 + enableLto = false; }; opencv4WithoutCuda = opencv4.override { diff --git a/pkgs/top-level/cuda-packages.nix b/pkgs/top-level/cuda-packages.nix index 639fa70446be..b2a23363707e 100644 --- a/pkgs/top-level/cuda-packages.nix +++ b/pkgs/top-level/cuda-packages.nix @@ -33,7 +33,9 @@ let attrsets customisation fixedPoints + lists strings + trivial versions ; # Backbone @@ -81,6 +83,45 @@ let nccl = final.callPackage ../development/cuda-modules/nccl { }; nccl-tests = final.callPackage ../development/cuda-modules/nccl-tests { }; + tests = + let + bools = [ + true + false + ]; + configs = { + openCVFirst = bools; + useOpenCVDefaultCuda = bools; + useTorchDefaultCuda = bools; + }; + builder = + { + openCVFirst, + useOpenCVDefaultCuda, + useTorchDefaultCuda, + }@config: + { + name = strings.concatStringsSep "-" ( + [ + "test" + (if openCVFirst then "opencv" else "torch") + ] + ++ lists.optionals (if openCVFirst then useOpenCVDefaultCuda else useTorchDefaultCuda) [ + "with-default-cuda" + ] + ++ [ + "then" + (if openCVFirst then "torch" else "opencv") + ] + ++ lists.optionals (if openCVFirst then useTorchDefaultCuda else useOpenCVDefaultCuda) [ + "with-default-cuda" + ] + ); + value = final.callPackage ../development/cuda-modules/tests/opencv-and-torch config; + }; + in + attrsets.listToAttrs (attrsets.mapCartesianProduct builder configs); + writeGpuTestPython = final.callPackage ../development/cuda-modules/write-gpu-test-python.nix { }; });