diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix new file mode 100644 index 000000000000..952f7435c069 --- /dev/null +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -0,0 +1,44 @@ +{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }: + +stdenv.mkDerivation rec { + name = "arrow-cpp-${version}"; + version = "0.9.0"; + + src = fetchurl { + url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; + sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy"; + }; + + sourceRoot = "apache-arrow-${version}/cpp"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ boost python.pkgs.python python.pkgs.numpy ]; + + preConfigure = '' + substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + ''; + + BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; }; + FLATBUFFERS_HOME = flatbuffers; + GTEST_HOME = gtest; + GFLAGS_HOME = gflags; + LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; }; + RAPIDJSON_HOME = rapidjson; + SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; + ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; + ZSTD_HOME = zstd; + + cmakeFlags = [ + "-DARROW_PYTHON=ON" + ]; + + meta = { + description = "A cross-language development platform for in-memory data"; + homepage = https://arrow.apache.org/; + license = stdenv.lib.licenses.asl20; + platforms = stdenv.lib.platforms.unix; + maintainers = with stdenv.lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/development/libraries/parquet-cpp/default.nix b/pkgs/development/libraries/parquet-cpp/default.nix new file mode 100644 index 000000000000..e281e604380b --- /dev/null +++ b/pkgs/development/libraries/parquet-cpp/default.nix @@ -0,0 +1,37 @@ +{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }: + +stdenv.mkDerivation rec { + name = "parquet-cpp-${version}"; + version = "1.4.0"; + + src = fetchurl { + url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz"; + sha256 = "1kn7pjzi5san5f05qbl8l8znqsa3f9cq9bflfr4s2jfwr7k9p2aj"; + }; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ boost ]; + + preConfigure = '' + substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY + ''; + + ARROW_HOME = arrow-cpp; + THRIFT_HOME = thrift; + GTEST_HOME = gtest; + SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; }; + ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; }; + + cmakeFlags = [ + "-DPARQUET_BUILD_BENCHMARKS=OFF" + ]; + + meta = { + description = "A C++ library to read and write the Apache Parquet columnar data format"; + homepage = http://parquet.apache.org; + license = stdenv.lib.licenses.asl20; + platforms = stdenv.lib.platforms.unix; + maintainers = with stdenv.lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/development/libraries/rapidjson/default.nix b/pkgs/development/libraries/rapidjson/default.nix index 8e6b3b44491e..a1671aa3e0f9 100644 --- a/pkgs/development/libraries/rapidjson/default.nix +++ b/pkgs/development/libraries/rapidjson/default.nix @@ -13,14 +13,16 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ pkgconfig cmake ]; - # detected by gcc7 - NIX_CFLAGS_COMPILE = [ "-Wno-error=implicit-fallthrough" ]; + preConfigure = '' + substituteInPlace CMakeLists.txt --replace "-Werror" "" + substituteInPlace example/CMakeLists.txt --replace "-Werror" "" + ''; meta = with lib; { description = "Fast JSON parser/generator for C++ with both SAX/DOM style API"; homepage = "http://rapidjson.org/"; license = licenses.mit; - platforms = platforms.linux; + platforms = platforms.unix; maintainers = with maintainers; [ cstrahan ]; }; } diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix new file mode 100644 index 000000000000..1c2cb4a7643d --- /dev/null +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -0,0 +1,65 @@ +{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }: + +let + _arrow-cpp = arrow-cpp.override { inherit python;}; + _parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; }; +in + +buildPythonPackage rec { + pname = "pyarrow"; + version = "0.9.0"; + + src = fetchurl { + url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; + sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy"; + }; + + sourceRoot = "apache-arrow-${version}/python"; + + nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ]; + propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ]; + checkInputs = [ pandas pytest pytestrunner ]; + + PYARROW_BUILD_TYPE = "release"; + PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib"; + + preBuild = '' + substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"' + substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"' + + # fix the hardcoded value + substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")' + ''; + + preCheck = '' + rm pyarrow/tests/test_hdfs.py + + # fails: "ArrowNotImplementedError: Unsupported numpy type 22" + substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled" + + # runs out of memory on @grahamcofborg linux box + substituteInPlace pyarrow/tests/test_feather.py --replace "test_large_dataframe" "_disabled" + + # probably broken on python2 + substituteInPlace pyarrow/tests/test_feather.py --replace "test_unicode_filename" "_disabled" + + # fails "error: [Errno 2] No such file or directory: 'test'" because + # nix_run_setup invocation somehow manages to import deserialize_buffer.py + # when it is not intended to be imported at all + rm pyarrow/tests/deserialize_buffer.py + substituteInPlace pyarrow/tests/test_feather.py --replace "test_deserialize_buffer_in_different_process" "_disabled" + ''; + + ARROW_HOME = _arrow-cpp; + PARQUET_HOME = _parquet-cpp; + + setupPyBuildFlags = ["--with-parquet" ]; + + meta = with lib; { + description = "A cross-language development platform for in-memory data"; + homepage = https://arrow.apache.org/; + license = lib.licenses.asl20; + platforms = platforms.unix; + maintainers = with lib.maintainers; [ veprbl ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7e0c3ca25693..2b5b02401f42 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -8405,6 +8405,8 @@ with pkgs; armadillo = callPackage ../development/libraries/armadillo {}; + arrow-cpp = callPackage ../development/libraries/arrow-cpp {}; + assimp = callPackage ../development/libraries/assimp { }; asio = callPackage ../development/libraries/asio { }; @@ -10876,6 +10878,8 @@ with pkgs; paperkey = callPackage ../tools/security/paperkey { }; + parquet-cpp = callPackage ../development/libraries/parquet-cpp {}; + pangoxsl = callPackage ../development/libraries/pangoxsl { }; pcaudiolib = callPackage ../development/libraries/pcaudiolib { diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 9d98e11783af..93e821fb719d 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -311,6 +311,10 @@ in { pyamf = callPackage ../development/python-modules/pyamf { }; + pyarrow = callPackage ../development/python-modules/pyarrow { + inherit (pkgs) arrow-cpp cmake pkgconfig; + }; + pyatspi = disabledIf (!isPy3k) (callPackage ../development/python-modules/pyatspi { }); pyaxmlparser = callPackage ../development/python-modules/pyaxmlparser { };