From af86bf34ff2d7a6e31280921461a241f4b043aac Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Fri, 16 Mar 2018 16:05:56 -0400 Subject: [PATCH 1/6] rapidjson: fix build on darwin Disables -Werror Alternatively one could use NIX_CFLAGS_COMPILE = [ "-Wno-error=implicit-fallthrough" "-Wno-error=zero-as-null-pointer-constant" "-Wno-error=shadow" ]; --- pkgs/development/libraries/rapidjson/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/development/libraries/rapidjson/default.nix b/pkgs/development/libraries/rapidjson/default.nix index 8e6b3b44491e..a1671aa3e0f9 100644 --- a/pkgs/development/libraries/rapidjson/default.nix +++ b/pkgs/development/libraries/rapidjson/default.nix @@ -13,14 +13,16 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ pkgconfig cmake ]; - # detected by gcc7 - NIX_CFLAGS_COMPILE = [ "-Wno-error=implicit-fallthrough" ]; + preConfigure = '' + substituteInPlace CMakeLists.txt --replace "-Werror" "" + substituteInPlace example/CMakeLists.txt --replace "-Werror" "" + ''; meta = with lib; { description = "Fast JSON parser/generator for C++ with both SAX/DOM style API"; homepage = "http://rapidjson.org/"; license = licenses.mit; - platforms = platforms.linux; + platforms = platforms.unix; maintainers = with maintainers; [ cstrahan ]; }; } From c5df226c738d26944487914d11a3531745cdd79a Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Fri, 16 Mar 2018 16:19:24 -0400 Subject: [PATCH 2/6] arrow-cpp: init at 0.8.0 --- .../libraries/arrow-cpp/default.nix | 40 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 42 insertions(+) create mode 100644 pkgs/development/libraries/arrow-cpp/default.nix diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix new file mode 100644 index 000000000000..7b7099cb610e --- /dev/null +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -0,0 +1,40 @@ +{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, rapidjson, snappy, zlib, zstd }: + +stdenv.mkDerivation rec { + name = "arrow-cpp-${version}"; + version = "0.8.0"; + + src = fetchurl { + url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; + sha256 = "1i79sh9ip32agbrn4n51pjn9266i45s8spk5jsi8ax0hqy1vhhmi"; + }; + + sourceRoot = "apache-arrow-${version}/cpp"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ boost ]; + + preConfigure = '' + substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + ''; + + BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; }; + FLATBUFFERS_HOME = flatbuffers; + GTEST_HOME = gtest; + GFLAGS_HOME = gflags; + LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; }; + RAPIDJSON_HOME = rapidjson; + SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; + ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; + ZSTD_HOME = zstd; + + meta = { + description = "A cross-language development platform for in-memory data"; + homepage = https://arrow.apache.org/; + license = stdenv.lib.licenses.asl20; + platforms = stdenv.lib.platforms.unix; + maintainers = with stdenv.lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 62ad5f27dfe7..9ffabd197861 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -8379,6 +8379,8 @@ with pkgs; armadillo = callPackage ../development/libraries/armadillo {}; + arrow-cpp = callPackage ../development/libraries/arrow-cpp {}; + assimp = callPackage ../development/libraries/assimp { }; asio = callPackage ../development/libraries/asio { }; From f9192c542467085dee4b817ea753ae40916cbf62 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 18 Mar 2018 20:13:16 -0400 Subject: [PATCH 3/6] pythonPackages.pyarrow: init at 0.8.0 --- .../libraries/arrow-cpp/default.nix | 8 +++- .../python-modules/pyarrow/default.nix | 47 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 4 ++ 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/python-modules/pyarrow/default.nix diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index 7b7099cb610e..1db034170ddc 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -1,4 +1,4 @@ -{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, rapidjson, snappy, zlib, zstd }: +{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, pythonPackages, rapidjson, snappy, zlib, zstd }: stdenv.mkDerivation rec { name = "arrow-cpp-${version}"; @@ -12,7 +12,7 @@ stdenv.mkDerivation rec { sourceRoot = "apache-arrow-${version}/cpp"; nativeBuildInputs = [ cmake ]; - buildInputs = [ boost ]; + buildInputs = [ boost pythonPackages.python pythonPackages.numpy ]; preConfigure = '' substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY @@ -30,6 +30,10 @@ stdenv.mkDerivation rec { ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; ZSTD_HOME = zstd; + cmakeFlags = [ + "-DARROW_PYTHON=ON" + ]; + meta = { description = "A cross-language development platform for in-memory data"; homepage = https://arrow.apache.org/; diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix new file mode 100644 index 000000000000..8804953045e4 --- /dev/null +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -0,0 +1,47 @@ +{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }: + +buildPythonPackage rec { + pname = "pyarrow"; + version = "0.8.0"; + + src = fetchurl { + url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; + sha256 = "1i79sh9ip32agbrn4n51pjn9266i45s8spk5jsi8ax0hqy1vhhmi"; + }; + + sourceRoot = "apache-arrow-${version}/python"; + + nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ]; + propagatedBuildInputs = [ futures numpy six ]; + checkInputs = [ pandas pytest ]; + + PYARROW_BUILD_TYPE = "release"; + PYARROW_BUNDLE_ARROW_CPP = 1; # sets RPATH on darwin + + preBuild = '' + substituteInPlace CMakeLists.txt --replace "''${ARROW_SO_VERSION}" '"0"' + ''; + + preCheck = '' + rm pyarrow/tests/test_hdfs.py + + # fails: "ArrowNotImplementedError: Unsupported numpy type 22" + substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled" + + # runs out of memory on @grahamcofborg linux box + substituteInPlace pyarrow/tests/test_feather.py --replace "test_large_dataframe" "_disabled" + + # probably broken on python2 + substituteInPlace pyarrow/tests/test_feather.py --replace "test_unicode_filename" "_disabled" + ''; + + ARROW_HOME = arrow-cpp; + + meta = with lib; { + description = "A cross-language development platform for in-memory data"; + homepage = https://arrow.apache.org/; + license = lib.licenses.asl20; + platforms = platforms.unix; + maintainers = with lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 85179ccce3f8..635b779ce29c 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -305,6 +305,10 @@ in { pyamf = callPackage ../development/python-modules/pyamf { }; + pyarrow = callPackage ../development/python-modules/pyarrow { + inherit (pkgs) arrow-cpp cmake pkgconfig; + }; + pyatspi = disabledIf (!isPy3k) (callPackage ../development/python-modules/pyatspi { }); pyaxmlparser = callPackage ../development/python-modules/pyaxmlparser { }; From a49ebbb7435b08509f05e894d37e93a5498a9d69 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Tue, 20 Mar 2018 18:19:39 -0400 Subject: [PATCH 4/6] parquet-cpp: init at 1.4.0 Also fix RPATH issues in pyarrow --- .../libraries/parquet-cpp/default.nix | 37 +++++++++++++++++++ .../python-modules/pyarrow/default.nix | 13 +++++-- pkgs/top-level/all-packages.nix | 2 + 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 pkgs/development/libraries/parquet-cpp/default.nix diff --git a/pkgs/development/libraries/parquet-cpp/default.nix b/pkgs/development/libraries/parquet-cpp/default.nix new file mode 100644 index 000000000000..e281e604380b --- /dev/null +++ b/pkgs/development/libraries/parquet-cpp/default.nix @@ -0,0 +1,37 @@ +{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }: + +stdenv.mkDerivation rec { + name = "parquet-cpp-${version}"; + version = "1.4.0"; + + src = fetchurl { + url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz"; + sha256 = "1kn7pjzi5san5f05qbl8l8znqsa3f9cq9bflfr4s2jfwr7k9p2aj"; + }; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ boost ]; + + preConfigure = '' + substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + ''; + + ARROW_HOME = arrow-cpp; + THRIFT_HOME = thrift; + GTEST_HOME = gtest; + SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; + ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; + + cmakeFlags = [ + "-DPARQUET_BUILD_BENCHMARKS=OFF" + ]; + + meta = { + description = "A C++ library to read and write the Apache Parquet columnar data format"; + homepage = http://parquet.apache.org; + license = stdenv.lib.licenses.asl20; + platforms = stdenv.lib.platforms.unix; + maintainers = with stdenv.lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 8804953045e4..11c2868c3370 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }: +{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, parquet-cpp, pkgconfig, setuptools_scm, six }: buildPythonPackage rec { pname = "pyarrow"; @@ -16,10 +16,14 @@ buildPythonPackage rec { checkInputs = [ pandas pytest ]; PYARROW_BUILD_TYPE = "release"; - PYARROW_BUNDLE_ARROW_CPP = 1; # sets RPATH on darwin + PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib"; preBuild = '' - substituteInPlace CMakeLists.txt --replace "''${ARROW_SO_VERSION}" '"0"' + substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"' + substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"' + + # fix the hardcoded value + substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${parquet-cpp.version}")' ''; preCheck = '' @@ -36,6 +40,9 @@ buildPythonPackage rec { ''; ARROW_HOME = arrow-cpp; + PARQUET_HOME = parquet-cpp; + + setupPyBuildFlags = ["--with-parquet" ]; meta = with lib; { description = "A cross-language development platform for in-memory data"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 9ffabd197861..a81fe88b8a85 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10872,6 +10872,8 @@ with pkgs; paperkey = callPackage ../tools/security/paperkey { }; + parquet-cpp = callPackage ../development/libraries/parquet-cpp {}; + pangoxsl = callPackage ../development/libraries/pangoxsl { }; pcaudiolib = callPackage ../development/libraries/pcaudiolib { From cdf612bc0d86f02a9f2d837656366872ed4e9376 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 4 Apr 2018 14:35:47 -0400 Subject: [PATCH 5/6] arrow-cpp,pythonPackages.pyarrow: 0.8.0 -> 0.9.0 --- pkgs/development/libraries/arrow-cpp/default.nix | 4 ++-- .../development/python-modules/pyarrow/default.nix | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index 1db034170ddc..7704cfd1c53b 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -2,11 +2,11 @@ stdenv.mkDerivation rec { name = "arrow-cpp-${version}"; - version = "0.8.0"; + version = "0.9.0"; src = fetchurl { url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; - sha256 = "1i79sh9ip32agbrn4n51pjn9266i45s8spk5jsi8ax0hqy1vhhmi"; + sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy"; }; sourceRoot = "apache-arrow-${version}/cpp"; diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 11c2868c3370..5939e3a77949 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,19 +1,19 @@ -{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, parquet-cpp, pkgconfig, setuptools_scm, six }: +{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }: buildPythonPackage rec { pname = "pyarrow"; - version = "0.8.0"; + version = "0.9.0"; src = fetchurl { url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; - sha256 = "1i79sh9ip32agbrn4n51pjn9266i45s8spk5jsi8ax0hqy1vhhmi"; + sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy"; }; sourceRoot = "apache-arrow-${version}/python"; nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ]; propagatedBuildInputs = [ futures numpy six ]; - checkInputs = [ pandas pytest ]; + checkInputs = [ pandas pytest pytestrunner ]; PYARROW_BUILD_TYPE = "release"; PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib"; @@ -37,6 +37,12 @@ buildPythonPackage rec { # probably broken on python2 substituteInPlace pyarrow/tests/test_feather.py --replace "test_unicode_filename" "_disabled" + + # fails "error: [Errno 2] No such file or directory: 'test'" because + # nix_run_setup invocation somehow manages to import deserialize_buffer.py + # when it is not intended to be imported at all + rm pyarrow/tests/deserialize_buffer.py + substituteInPlace pyarrow/tests/test_feather.py --replace "test_deserialize_buffer_in_different_process" "_disabled" ''; ARROW_HOME = arrow-cpp; From 671e9bd3bf4a1e9ac56288f86e8d59db289911e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 8 Apr 2018 19:59:38 -0400 Subject: [PATCH 6/6] python.pkgs.pyarrow: fix for python3 --- pkgs/development/libraries/arrow-cpp/default.nix | 4 ++-- .../python-modules/pyarrow/default.nix | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index 7704cfd1c53b..952f7435c069 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -1,4 +1,4 @@ -{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, pythonPackages, rapidjson, snappy, zlib, zstd }: +{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }: stdenv.mkDerivation rec { name = "arrow-cpp-${version}"; @@ -12,7 +12,7 @@ stdenv.mkDerivation rec { sourceRoot = "apache-arrow-${version}/cpp"; nativeBuildInputs = [ cmake ]; - buildInputs = [ boost pythonPackages.python pythonPackages.numpy ]; + buildInputs = [ boost python.pkgs.python python.pkgs.numpy ]; preConfigure = '' substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 5939e3a77949..1c2cb4a7643d 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,4 +1,9 @@ -{ lib, buildPythonPackage, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }: +{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }: + +let + _arrow-cpp = arrow-cpp.override { inherit python;}; + _parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; }; +in buildPythonPackage rec { pname = "pyarrow"; @@ -12,7 +17,7 @@ buildPythonPackage rec { sourceRoot = "apache-arrow-${version}/python"; nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ]; - propagatedBuildInputs = [ futures numpy six ]; + propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ]; checkInputs = [ pandas pytest pytestrunner ]; PYARROW_BUILD_TYPE = "release"; @@ -23,7 +28,7 @@ buildPythonPackage rec { substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"' # fix the hardcoded value - substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${parquet-cpp.version}")' + substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")' ''; preCheck = '' @@ -45,8 +50,8 @@ buildPythonPackage rec { substituteInPlace pyarrow/tests/test_feather.py --replace "test_deserialize_buffer_in_different_process" "_disabled" ''; - ARROW_HOME = arrow-cpp; - PARQUET_HOME = parquet-cpp; + ARROW_HOME = _arrow-cpp; + PARQUET_HOME = _parquet-cpp; setupPyBuildFlags = ["--with-parquet" ];