You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2019/04/10 03:51:18 UTC
[arrow] branch master updated: ARROW-5124: [C++] Add support for
Parquet in MinGW build
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 2a4e48e ARROW-5124: [C++] Add support for Parquet in MinGW build
2a4e48e is described below
commit 2a4e48e384be618fc476e8ba8091642c7091f705
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Wed Apr 10 12:50:58 2019 +0900
ARROW-5124: [C++] Add support for Parquet in MinGW build
* cpp/cmake_modules/BuildUtils.cmake:
We need to compile sources for static and shared libraries
separately with MinGW too. It's for ARROW_STATIC and
PARQUET_STATIC definitions.
* cpp/cmake_modules/FindThrift.cmake:
Add missing ws2_32 link. We don't need this for Visual C++ because
Thrift uses "#pragma comment(lib, "Ws2_32.lib")" in
thrift/windows/config.h for Visual C++.
* cpp/src/parquet/types-test.cc:
We need to ignore deprecated-declarations warnings.
* cpp/src/parquet/util/comparison*:
We can't use extern template class with MinGW.
See also: https://issues.apache.org/jira/browse/ARROW-4399
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #4116 from kou/cpp-parquet-mingw and squashes the following commits:
96bedfad <Kouhei Sutou> Add a comment for Thrift and MinGW
aa6b6722 <Kouhei Sutou> Add support for Parquet in MinGW build
---
ci/appveyor-cpp-build-mingw.bat | 16 ++++++++++++++--
ci/appveyor-cpp-setup-mingw.bat | 4 ++--
cpp/cmake_modules/BuildUtils.cmake | 6 +++---
cpp/cmake_modules/FindThrift.cmake | 6 ++++++
cpp/src/parquet/types-test.cc | 4 ++--
cpp/src/parquet/util/comparison.cc | 9 ---------
cpp/src/parquet/util/comparison.h | 33 ++++++++++++---------------------
dev/release/verify-release-candidate.sh | 3 ---
ruby/red-arrow-cuda/Gemfile | 2 ++
ruby/red-arrow-cuda/test/run-test.rb | 4 ++++
ruby/red-gandiva/Gemfile | 2 ++
ruby/red-gandiva/test/run-test.rb | 4 ++++
ruby/red-parquet/Gemfile | 2 ++
ruby/red-parquet/test/run-test.rb | 4 ++++
ruby/red-plasma/Gemfile | 2 ++
ruby/red-plasma/test/run-test.rb | 4 ++++
16 files changed, 63 insertions(+), 42 deletions(-)
diff --git a/ci/appveyor-cpp-build-mingw.bat b/ci/appveyor-cpp-build-mingw.bat
index c92b8eb..552fd32 100644
--- a/ci/appveyor-cpp-build-mingw.bat
+++ b/ci/appveyor-cpp-build-mingw.bat
@@ -32,6 +32,10 @@ for /f "usebackq" %%v in (`python3 -c "import sys; print('.'.join(map(str, sys.v
set PYTHON_VERSION=%%v
)
+git submodule update --init || exit /B
+set ARROW_TEST_DATA=%CD%\testing\data
+set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data
+
set CPP_BUILD_DIR=cpp\build
mkdir %CPP_BUILD_DIR%
pushd %CPP_BUILD_DIR%
@@ -44,6 +48,7 @@ cmake ^
-DARROW_PACKAGE_PREFIX=%MINGW_PREFIX% ^
-DARROW_JEMALLOC=OFF ^
-DARROW_USE_GLOG=OFF ^
+ -DARROW_PARQUET=ON ^
-DARROW_PYTHON=ON ^
-DPythonInterp_FIND_VERSION=ON ^
-DPythonInterp_FIND_VERSION_MAJOR=3 ^
@@ -77,6 +82,13 @@ ninja -C %C_GLIB_BUILD_DIR% install || exit /B
ruby c_glib\test\run-test.rb || exit /B
pushd ruby\red-arrow
-ruby -S bundle install
-ruby -rdevkit test\run-test.rb
+ruby -S bundle install || exit /B
+pacman --remove --noconfirm "%MINGW_PACKAGE_PREFIX%-arrow" || exit /B
+ruby -rdevkit test\run-test.rb || exit /B
+popd
+
+pushd ruby\red-parquet
+ruby -S bundle install || exit /B
+pacman --remove --noconfirm "%MINGW_PACKAGE_PREFIX%-arrow" || exit /B
+ruby test\run-test.rb || exit /B
popd
diff --git a/ci/appveyor-cpp-setup-mingw.bat b/ci/appveyor-cpp-setup-mingw.bat
index c0c44ff..d65b9bd 100644
--- a/ci/appveyor-cpp-setup-mingw.bat
+++ b/ci/appveyor-cpp-setup-mingw.bat
@@ -24,13 +24,12 @@ if "%MSYSTEM%" == "MINGW32" (
)
set PATH=%MINGW_PREFIX%\bin;C:\msys64\usr\bin;%PATH%
-pacman -S --noconfirm ^
+pacman --sync --refresh --noconfirm ^
"%MINGW_PACKAGE_PREFIX%-boost" ^
"%MINGW_PACKAGE_PREFIX%-brotli" ^
"%MINGW_PACKAGE_PREFIX%-cmake" ^
"%MINGW_PACKAGE_PREFIX%-double-conversion" ^
"%MINGW_PACKAGE_PREFIX%-flatbuffers" ^
- "%MINGW_PACKAGE_PREFIX%-gcc" ^
"%MINGW_PACKAGE_PREFIX%-gflags" ^
"%MINGW_PACKAGE_PREFIX%-gobject-introspection" ^
"%MINGW_PACKAGE_PREFIX%-gtk-doc" ^
@@ -40,6 +39,7 @@ pacman -S --noconfirm ^
"%MINGW_PACKAGE_PREFIX%-python3-numpy" ^
"%MINGW_PACKAGE_PREFIX%-rapidjson" ^
"%MINGW_PACKAGE_PREFIX%-snappy" ^
+ "%MINGW_PACKAGE_PREFIX%-thrift" ^
"%MINGW_PACKAGE_PREFIX%-zlib" ^
"%MINGW_PACKAGE_PREFIX%-zstd" || exit /B
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index b318f8b..8b4e26b 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -165,9 +165,9 @@ function(ADD_ARROW_LIB LIB_NAME)
set(BUILD_STATIC ${ARROW_BUILD_STATIC})
endif()
- if(MSVC OR (CMAKE_GENERATOR STREQUAL Xcode))
- # MSVC needs to compile C++ separately for each library kind (shared and static)
- # because of dllexport declarations
+ if(WIN32 OR (CMAKE_GENERATOR STREQUAL Xcode))
+ # We need to compile C++ separately for each library kind (shared and static)
+ # because of dllexport declarations on Windows.
# The Xcode generator doesn't reliably work with Xcode as target names are not
# guessed correctly.
set(LIB_DEPS ${ARG_SOURCES})
diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake
index a4decf7..86fb229 100644
--- a/cpp/cmake_modules/FindThrift.cmake
+++ b/cpp/cmake_modules/FindThrift.cmake
@@ -103,6 +103,12 @@ if(Thrift_FOUND OR THRIFT_FOUND)
set_target_properties(Thrift::thrift
PROPERTIES IMPORTED_LOCATION "${THRIFT_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}")
+ if(WIN32 AND NOT MSVC)
+ # We don't need this for Visual C++ because Thrift uses
+ # "#pragma comment(lib, "Ws2_32.lib")" in
+ # thrift/windows/config.h for Visual C++.
+ set_target_properties(Thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES "ws2_32")
+ endif()
extract_thrift_version()
endif()
diff --git a/cpp/src/parquet/types-test.cc b/cpp/src/parquet/types-test.cc
index 6d1e9bd..b78d677 100644
--- a/cpp/src/parquet/types-test.cc
+++ b/cpp/src/parquet/types-test.cc
@@ -71,10 +71,10 @@ TEST(TestCompressionToString, Compression) {
ASSERT_STREQ("ZSTD", CompressionToString(Compression::ZSTD).c_str());
}
-#if !(defined(_WIN32) || defined(__CYGWIN__))
+#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif _MSC_VER
+#elif defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4996)
#endif
diff --git a/cpp/src/parquet/util/comparison.cc b/cpp/src/parquet/util/comparison.cc
index 99733fc..32c9b9f 100644
--- a/cpp/src/parquet/util/comparison.cc
+++ b/cpp/src/parquet/util/comparison.cc
@@ -64,15 +64,6 @@ std::shared_ptr<Comparator> Comparator::Make(const ColumnDescriptor* descr) {
return nullptr;
}
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<BooleanType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int32Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int64Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<Int96Type>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<FloatType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<DoubleType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<ByteArrayType>;
-template class PARQUET_TEMPLATE_EXPORT CompareDefault<FLBAType>;
-
bool CompareUnsignedInt32::operator()(const int32_t& a, const int32_t& b) {
const uint32_t ua = a;
const uint32_t ub = b;
diff --git a/cpp/src/parquet/util/comparison.h b/cpp/src/parquet/util/comparison.h
index 4daa4df..88c04bc 100644
--- a/cpp/src/parquet/util/comparison.h
+++ b/cpp/src/parquet/util/comparison.h
@@ -36,7 +36,7 @@ class PARQUET_EXPORT Comparator {
// The default comparison is SIGNED
template <typename DType>
-class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault : public Comparator {
+class CompareDefault : public Comparator {
public:
typedef typename DType::c_type T;
CompareDefault() {}
@@ -44,7 +44,7 @@ class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault : public Comparator {
};
template <>
-class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<Int96Type> : public Comparator {
+class CompareDefault<Int96Type> : public Comparator {
public:
CompareDefault() {}
virtual bool operator()(const Int96& a, const Int96& b) {
@@ -62,7 +62,7 @@ class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<Int96Type> : public Comparato
};
template <>
-class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<ByteArrayType> : public Comparator {
+class CompareDefault<ByteArrayType> : public Comparator {
public:
CompareDefault() {}
virtual bool operator()(const ByteArray& a, const ByteArray& b) {
@@ -73,7 +73,7 @@ class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<ByteArrayType> : public Compa
};
template <>
-class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<FLBAType> : public Comparator {
+class CompareDefault<FLBAType> : public Comparator {
public:
explicit CompareDefault(int length) : type_length_(length) {}
virtual bool operator()(const FLBA& a, const FLBA& b) {
@@ -85,14 +85,14 @@ class PARQUET_TEMPLATE_CLASS_EXPORT CompareDefault<FLBAType> : public Comparator
int32_t type_length_;
};
-typedef CompareDefault<BooleanType> CompareDefaultBoolean;
-typedef CompareDefault<Int32Type> CompareDefaultInt32;
-typedef CompareDefault<Int64Type> CompareDefaultInt64;
-typedef CompareDefault<Int96Type> CompareDefaultInt96;
-typedef CompareDefault<FloatType> CompareDefaultFloat;
-typedef CompareDefault<DoubleType> CompareDefaultDouble;
-typedef CompareDefault<ByteArrayType> CompareDefaultByteArray;
-typedef CompareDefault<FLBAType> CompareDefaultFLBA;
+using CompareDefaultBoolean = CompareDefault<BooleanType>;
+using CompareDefaultInt32 = CompareDefault<Int32Type>;
+using CompareDefaultInt64 = CompareDefault<Int64Type>;
+using CompareDefaultInt96 = CompareDefault<Int96Type>;
+using CompareDefaultFloat = CompareDefault<FloatType>;
+using CompareDefaultDouble = CompareDefault<DoubleType>;
+using CompareDefaultByteArray = CompareDefault<ByteArrayType>;
+using CompareDefaultFLBA = CompareDefault<FLBAType>;
// Define Unsigned Comparators
class PARQUET_EXPORT CompareUnsignedInt32 : public CompareDefaultInt32 {
@@ -121,15 +121,6 @@ class PARQUET_EXPORT CompareUnsignedFLBA : public CompareDefaultFLBA {
bool operator()(const FLBA& a, const FLBA& b) override;
};
-PARQUET_EXTERN_TEMPLATE CompareDefault<BooleanType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int32Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int64Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<Int96Type>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<FloatType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<DoubleType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<ByteArrayType>;
-PARQUET_EXTERN_TEMPLATE CompareDefault<FLBAType>;
-
} // namespace parquet
#endif // PARQUET_UTIL_COMPARISON_H
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 40dee92..a7d2013 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -339,9 +339,6 @@ test_ruby() {
for module in ${modules}; do
pushd ${module}
- if [ "${module}" != "red-arrow" ]; then
- echo 'gem "red-arrow", path: "../red-arrow"' >> Gemfile
- fi
bundle install --path vendor/bundle
bundle exec ruby test/run-test.rb
popd
diff --git a/ruby/red-arrow-cuda/Gemfile b/ruby/red-arrow-cuda/Gemfile
index 3907918..7c4cefc 100644
--- a/ruby/red-arrow-cuda/Gemfile
+++ b/ruby/red-arrow-cuda/Gemfile
@@ -20,3 +20,5 @@
source "https://rubygems.org/"
gemspec
+
+gem "red-arrow", path: "../red-arrow"
diff --git a/ruby/red-arrow-cuda/test/run-test.rb b/ruby/red-arrow-cuda/test/run-test.rb
index a4f7f76..7c3efa4 100755
--- a/ruby/red-arrow-cuda/test/run-test.rb
+++ b/ruby/red-arrow-cuda/test/run-test.rb
@@ -21,6 +21,10 @@ $VERBOSE = true
require "pathname"
+(ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
+ RubyInstaller::Runtime.add_dll_directory(path)
+end
+
base_dir = Pathname.new(__dir__).parent.expand_path
arrow_base_dir = base_dir.parent + "red-arrow"
diff --git a/ruby/red-gandiva/Gemfile b/ruby/red-gandiva/Gemfile
index 3907918..7c4cefc 100644
--- a/ruby/red-gandiva/Gemfile
+++ b/ruby/red-gandiva/Gemfile
@@ -20,3 +20,5 @@
source "https://rubygems.org/"
gemspec
+
+gem "red-arrow", path: "../red-arrow"
diff --git a/ruby/red-gandiva/test/run-test.rb b/ruby/red-gandiva/test/run-test.rb
index a4f7f76..7c3efa4 100755
--- a/ruby/red-gandiva/test/run-test.rb
+++ b/ruby/red-gandiva/test/run-test.rb
@@ -21,6 +21,10 @@ $VERBOSE = true
require "pathname"
+(ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
+ RubyInstaller::Runtime.add_dll_directory(path)
+end
+
base_dir = Pathname.new(__dir__).parent.expand_path
arrow_base_dir = base_dir.parent + "red-arrow"
diff --git a/ruby/red-parquet/Gemfile b/ruby/red-parquet/Gemfile
index 3907918..7c4cefc 100644
--- a/ruby/red-parquet/Gemfile
+++ b/ruby/red-parquet/Gemfile
@@ -20,3 +20,5 @@
source "https://rubygems.org/"
gemspec
+
+gem "red-arrow", path: "../red-arrow"
diff --git a/ruby/red-parquet/test/run-test.rb b/ruby/red-parquet/test/run-test.rb
index a4f7f76..7c3efa4 100755
--- a/ruby/red-parquet/test/run-test.rb
+++ b/ruby/red-parquet/test/run-test.rb
@@ -21,6 +21,10 @@ $VERBOSE = true
require "pathname"
+(ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
+ RubyInstaller::Runtime.add_dll_directory(path)
+end
+
base_dir = Pathname.new(__dir__).parent.expand_path
arrow_base_dir = base_dir.parent + "red-arrow"
diff --git a/ruby/red-plasma/Gemfile b/ruby/red-plasma/Gemfile
index 3907918..7c4cefc 100644
--- a/ruby/red-plasma/Gemfile
+++ b/ruby/red-plasma/Gemfile
@@ -20,3 +20,5 @@
source "https://rubygems.org/"
gemspec
+
+gem "red-arrow", path: "../red-arrow"
diff --git a/ruby/red-plasma/test/run-test.rb b/ruby/red-plasma/test/run-test.rb
index a4f7f76..7c3efa4 100755
--- a/ruby/red-plasma/test/run-test.rb
+++ b/ruby/red-plasma/test/run-test.rb
@@ -21,6 +21,10 @@ $VERBOSE = true
require "pathname"
+(ENV["ARROW_DLL_PATH"] || "").split(File::PATH_SEPARATOR).each do |path|
+ RubyInstaller::Runtime.add_dll_directory(path)
+end
+
base_dir = Pathname.new(__dir__).parent.expand_path
arrow_base_dir = base_dir.parent + "red-arrow"