You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ra...@apache.org on 2019/04/04 08:53:22 UTC
[arrow] branch master updated: ARROW-4959:[C++][Gandiva][Crossbow]
Gandiva crossbow packaging changes.
This is an automated email from the ASF dual-hosted git repository.
ravindra pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ba9f809 ARROW-4959:[C++][Gandiva][Crossbow] Gandiva crossbow packaging changes.
ba9f809 is described below
commit ba9f809e3c7c194e5c46bb1ff57c51ef3049aa7a
Author: Praveen <pr...@dremio.com>
AuthorDate: Thu Apr 4 14:23:05 2019 +0530
ARROW-4959:[C++][Gandiva][Crossbow] Gandiva crossbow packaging changes.
- Supports building Gandiva in the manylinux docker image.
- Fixes some issues discovered.
- Move crossbow to build using manylinux docker image.
- Use system dependencies to package Gandiva in OsX.
Author: Praveen <pr...@dremio.com>
Closes #4082 from praveenbingo/ARROW-4959-CentOS5 and squashes the following commits:
e5913de2 <Praveen> Fix review comments.
d5c6e5c3 <Praveen> Address review comments.
ab73e1ee <Praveen> Fix format issues.
936f05a7 <Praveen> Fix ci issues.
f0de7f63 <Praveen> Remove unnecessary files.
7627e282 <Praveen> Fix path for osx.
9cfc2023 <Praveen> Copy dylib.
fba3b4a8 <Praveen> Add a dummy target.
bb02b5ee <Praveen> Support both Java7 and Java8 environments.
b6b4a9ad <Praveen> Try OSX fixes.
aa13c3bb <Praveen> Fix java path.
a92841bf <Praveen> Fix path.
5e9c86b8 <Praveen> Fix paths.
5447d115 <Praveen> Misc fixes.
5880d1ca <Praveen> ARROW-4959: Package gandiva in manylinux docker.
---
cpp/src/gandiva/jni/CMakeLists.txt | 1 +
cpp/src/gandiva/jni/expression_registry_helper.cc | 4 +-
.../gandiva/precompiled/extended_math_ops_test.cc | 56 +++++++++++--------
dev/tasks/gandiva-jars/README.md | 29 ++++++++++
dev/tasks/gandiva-jars/build-cpp-linux.sh | 64 ++++++++++++++++++++++
.../{build-cpp.sh => build-cpp-osx.sh} | 6 +-
dev/tasks/gandiva-jars/build-java.sh | 8 +--
dev/tasks/gandiva-jars/travis.linux.yml | 10 ++--
dev/tasks/gandiva-jars/travis.osx.yml | 9 ++-
java/gandiva/CMakeLists.txt | 46 +++++++++++++---
.../arrow/gandiva/evaluator/ProjectorTest.java | 3 +-
11 files changed, 185 insertions(+), 51 deletions(-)
diff --git a/cpp/src/gandiva/jni/CMakeLists.txt b/cpp/src/gandiva/jni/CMakeLists.txt
index af58efd..680b6e5 100644
--- a/cpp/src/gandiva/jni/CMakeLists.txt
+++ b/cpp/src/gandiva/jni/CMakeLists.txt
@@ -74,6 +74,7 @@ add_arrow_lib(gandiva_jni
DEPENDENCIES
gandiva_java
gandiva_jni_proto
+ gandiva_jni_headers
EXTRA_INCLUDES
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include>
diff --git a/cpp/src/gandiva/jni/expression_registry_helper.cc b/cpp/src/gandiva/jni/expression_registry_helper.cc
index 94c4833..2275641 100644
--- a/cpp/src/gandiva/jni/expression_registry_helper.cc
+++ b/cpp/src/gandiva/jni/expression_registry_helper.cc
@@ -150,7 +150,7 @@ Java_org_apache_arrow_gandiva_evaluator_ExpressionRegistryJniHelper_getGandivaSu
types::ExtGandivaType* gandiva_data_type = gandiva_data_types.add_datatype();
ArrowToProtobuf(type, gandiva_data_type);
}
- int size = static_cast<int>(gandiva_data_types.ByteSizeLong());
+ int size = gandiva_data_types.ByteSize();
std::unique_ptr<jbyte[]> buffer{new jbyte[size]};
gandiva_data_types.SerializeToArray(reinterpret_cast<void*>(buffer.get()), size);
jbyteArray ret = env->NewByteArray(size);
@@ -179,7 +179,7 @@ Java_org_apache_arrow_gandiva_evaluator_ExpressionRegistryJniHelper_getGandivaSu
ArrowToProtobuf(param_type, proto_param_type);
}
}
- int size = static_cast<int>(gandiva_functions.ByteSizeLong());
+ int size = gandiva_functions.ByteSize();
std::unique_ptr<jbyte[]> buffer{new jbyte[size]};
gandiva_functions.SerializeToArray(reinterpret_cast<void*>(buffer.get()), size);
jbyteArray ret = env->NewByteArray(size);
diff --git a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
index fc9fadd..4f91260 100644
--- a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
+++ b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc
@@ -15,67 +15,75 @@
// specific language governing permissions and limitations
// under the License.
+#include <math.h>
+
#include <gtest/gtest.h>
#include "gandiva/execution_context.h"
#include "gandiva/precompiled/types.h"
namespace gandiva {
+static const double MAX_ERROR = 0.00005;
+
+void VerifyFuzzyEquals(double actual, double expected, double max_error = MAX_ERROR) {
+ EXPECT_TRUE(fabs(actual - expected) < max_error) << actual << " != " << expected;
+}
+
TEST(TestExtendedMathOps, TestCbrt) {
- EXPECT_EQ(cbrt_int32(27), 3);
- EXPECT_EQ(cbrt_int64(27), 3);
- EXPECT_EQ(cbrt_float32(27), 3);
- EXPECT_EQ(cbrt_float64(27), 3);
- EXPECT_EQ(cbrt_float64(-27), -3);
-
- EXPECT_EQ(cbrt_float32(15.625), 2.5);
- EXPECT_EQ(cbrt_float64(15.625), 2.5);
+ VerifyFuzzyEquals(cbrt_int32(27), 3);
+ VerifyFuzzyEquals(cbrt_int64(27), 3);
+ VerifyFuzzyEquals(cbrt_float32(27), 3);
+ VerifyFuzzyEquals(cbrt_float64(27), 3);
+ VerifyFuzzyEquals(cbrt_float64(-27), -3);
+
+ VerifyFuzzyEquals(cbrt_float32(15.625), 2.5);
+ VerifyFuzzyEquals(cbrt_float64(15.625), 2.5);
}
TEST(TestExtendedMathOps, TestExp) {
double val = 20.085536923187668;
- EXPECT_EQ(exp_int32(3), val);
- EXPECT_EQ(exp_int64(3), val);
- EXPECT_EQ(exp_float32(3), val);
- EXPECT_EQ(exp_float64(3), val);
+ VerifyFuzzyEquals(exp_int32(3), val);
+ VerifyFuzzyEquals(exp_int64(3), val);
+ VerifyFuzzyEquals(exp_float32(3), val);
+ VerifyFuzzyEquals(exp_float64(3), val);
}
TEST(TestExtendedMathOps, TestLog) {
double val = 4.1588830833596715;
- EXPECT_EQ(log_int32(64), val);
- EXPECT_EQ(log_int64(64), val);
- EXPECT_EQ(log_float32(64), val);
- EXPECT_EQ(log_float64(64), val);
+ VerifyFuzzyEquals(log_int32(64), val);
+ VerifyFuzzyEquals(log_int64(64), val);
+ VerifyFuzzyEquals(log_float32(64), val);
+ VerifyFuzzyEquals(log_float64(64), val);
EXPECT_EQ(log_int32(0), -std::numeric_limits<double>::infinity());
}
TEST(TestExtendedMathOps, TestLog10) {
- EXPECT_EQ(log10_int32(100), 2);
- EXPECT_EQ(log10_int64(100), 2);
- EXPECT_EQ(log10_float32(100), 2);
- EXPECT_EQ(log10_float64(100), 2);
+ VerifyFuzzyEquals(log10_int32(100), 2);
+ VerifyFuzzyEquals(log10_int64(100), 2);
+ VerifyFuzzyEquals(log10_float32(100), 2);
+ VerifyFuzzyEquals(log10_float64(100), 2);
}
TEST(TestExtendedMathOps, TestPower) {
- EXPECT_EQ(power_float64_float64(2, 5.4), 42.22425314473263);
- EXPECT_EQ(power_float64_float64(5.4, 2), 29.160000000000004);
+ VerifyFuzzyEquals(power_float64_float64(2, 5.4), 42.22425314473263);
+ VerifyFuzzyEquals(power_float64_float64(5.4, 2), 29.160000000000004);
}
TEST(TestExtendedMathOps, TestLogWithBase) {
gandiva::ExecutionContext context;
float64 out =
log_int32_int32(reinterpret_cast<int64>(&context), 1 /*base*/, 10 /*value*/);
- EXPECT_EQ(out, 0);
+ VerifyFuzzyEquals(out, 0);
EXPECT_EQ(context.has_error(), true);
EXPECT_TRUE(context.get_error().find("divide by zero error") != std::string::npos)
<< context.get_error();
gandiva::ExecutionContext context1;
out = log_int32_int32(reinterpret_cast<int64>(&context), 2 /*base*/, 64 /*value*/);
- EXPECT_EQ(out, 6);
+ VerifyFuzzyEquals(out, 6);
EXPECT_EQ(context1.has_error(), false);
}
diff --git a/dev/tasks/gandiva-jars/README.md b/dev/tasks/gandiva-jars/README.md
new file mode 100644
index 0000000..2f4c694
--- /dev/null
+++ b/dev/tasks/gandiva-jars/README.md
@@ -0,0 +1,29 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Updating manylinux for Gandiva Jar Build.
+
+Do the following to update arrow manylinux docker image for building Gandiva Jars
+
+- Install java in the manylinux image.
+- To do above, update Dockerfile-x86_64_base under python/manylinux1 to install java.
+- Please note only upto java7 is available in CentOS5, so install java7 in the base.
+- Export JAVA_HOME environment variable.
+- Then update build_boost.sh under python/manylinux1/scripts to build boost statically.
+
+Please look at https://github.com/praveenbingo/arrow/tree/buildGandivaDocker that already has these changes.
\ No newline at end of file
diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh
new file mode 100755
index 0000000..e1df5fc
--- /dev/null
+++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source /multibuild/manylinux_utils.sh
+
+# Quit on failure
+set -e
+
+# Print commands for debugging
+set -x
+
+ARROW_BUILD_DIR=/tmp/arrow-build
+mkdir -p "${ARROW_BUILD_DIR}"
+pushd "${ARROW_BUILD_DIR}"
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+ -DARROW_DEPENDENCY_SOURCE="SYSTEM" \
+ -DZLIB_ROOT=/usr/local \
+ -DCMAKE_INSTALL_PREFIX=/arrow-dist \
+ -DCMAKE_INSTALL_LIBDIR=lib \
+ -DARROW_BUILD_TESTS=ON \
+ -DARROW_BUILD_SHARED=ON \
+ -DARROW_BOOST_USE_SHARED=OFF \
+ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \
+ -DARROW_JEMALLOC=ON \
+ -DARROW_RPATH_ORIGIN=ON \
+ -DARROW_PYTHON=OFF \
+ -DARROW_PARQUET=OFF \
+ -DARROW_GANDIVA=ON \
+ -DARROW_GANDIVA_JAVA=ON \
+ -DARROW_GANDIVA_JAVA7=ON \
+ -DBoost_NAMESPACE=arrow_boost \
+ -Ddouble-conversion_SOURCE=BUNDLED \
+ -Dgflags_SOURCE=BUNDLED \
+ -DRapidJSON_SOURCE=BUNDLED \
+ -DRE2_SOURCE=BUNDLED \
+ -DARROW_BUILD_UTILITIES=OFF \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT=/arrow_boost_dist \
+ -GNinja /arrow/cpp
+ninja install
+ninja test
+popd
+
+
+# copy the library to distribution
+cp -L /arrow-dist/lib/libgandiva_jni.so /io/dist
+
diff --git a/dev/tasks/gandiva-jars/build-cpp.sh b/dev/tasks/gandiva-jars/build-cpp-osx.sh
similarity index 94%
rename from dev/tasks/gandiva-jars/build-cpp.sh
rename to dev/tasks/gandiva-jars/build-cpp-osx.sh
index b585b9d..8272943 100755
--- a/dev/tasks/gandiva-jars/build-cpp.sh
+++ b/dev/tasks/gandiva-jars/build-cpp-osx.sh
@@ -19,10 +19,10 @@
set -e
-source arrow/ci/travis_env_common.sh
+set -x
# Builds arrow + gandiva and tests the same.
-pushd arrow/cpp
+pushd cpp
mkdir build
pushd build
CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
@@ -40,5 +40,7 @@ pushd arrow/cpp
cmake $CMAKE_FLAGS ..
make -j4
ctest
+
+ cp -L release/libgandiva_jni.dylib $TRAVIS_BUILD_DIR/dist
popd
popd
diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh
index d099035..bbbf74a 100755
--- a/dev/tasks/gandiva-jars/build-java.sh
+++ b/dev/tasks/gandiva-jars/build-java.sh
@@ -19,11 +19,9 @@
set -e
-source arrow/ci/travis_env_common.sh
+CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/dist/
-CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp/build/release
-
-pushd arrow/java
+pushd java
if [ $TRAVIS_OS_NAME == "linux" ]; then
ldd $CPP_BUILD_DIR/libgandiva_jni.so
fi
@@ -33,5 +31,5 @@ pushd arrow/java
# test only gandiva
mvn test -P gandiva -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR
# copy the jars to distribution folder
- find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} ../../dist/ \;
+ find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} $CPP_BUILD_DIR \;
popd
diff --git a/dev/tasks/gandiva-jars/travis.linux.yml b/dev/tasks/gandiva-jars/travis.linux.yml
index 7c5eb33..1e87e3e 100644
--- a/dev/tasks/gandiva-jars/travis.linux.yml
+++ b/dev/tasks/gandiva-jars/travis.linux.yml
@@ -44,14 +44,14 @@ before_script:
- git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow
- git -C arrow checkout {{ arrow.head }}
- export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
- - arrow/ci/travis_install_linux.sh
- - arrow/ci/travis_install_clang_tools.sh
- - arrow/ci/travis_install_toolchain.sh
+ -
script:
+ - cd $TRAVIS_BUILD_DIR
- mkdir -p dist
- - arrow/dev/tasks/gandiva-jars/build-cpp.sh || travis_terminate 1
- - arrow/dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1
+ # please refer to README for steps to update this image
+ - docker run -v $PWD:/io -v $PWD:/arrow quay.io/praveenbingo/arrow:buildGandivaDocker /io/dev/tasks/gandiva-jars/build-cpp-linux.sh || travis_terminate 1
+ - dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1
deploy:
provider: releases
diff --git a/dev/tasks/gandiva-jars/travis.osx.yml b/dev/tasks/gandiva-jars/travis.osx.yml
index b801725..b3139f2 100644
--- a/dev/tasks/gandiva-jars/travis.osx.yml
+++ b/dev/tasks/gandiva-jars/travis.osx.yml
@@ -36,12 +36,15 @@ before_script:
- git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow
- git -C arrow checkout {{ arrow.head }}
- export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
- - arrow/ci/travis_install_toolchain.sh
+ - brew update
+ - brew upgrade cmake
+ - brew install bison flex llvm@7
script:
+ - cd arrow
- mkdir -p dist
- - arrow/dev/tasks/gandiva-jars/build-cpp.sh || travis_terminate 1
- - arrow/dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1
+ - dev/tasks/gandiva-jars/build-cpp-osx.sh || travis_terminate 1
+ - dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1
deploy:
provider: releases
diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt
index 815acea..0a7c4d0 100644
--- a/java/gandiva/CMakeLists.txt
+++ b/java/gandiva/CMakeLists.txt
@@ -23,12 +23,40 @@ include(UseJava)
include(FindJNI)
message("generating headers to ${JNI_HEADERS_DIR}/jni")
-add_jar(gandiva_java
- src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java
- src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java
- src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java
- src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java
- GENERATE_NATIVE_HEADERS
- gandivajni_headers
- DESTINATION
- ${JNI_HEADERS_DIR}/jni)
+
+# generate_native_headers is available only from java8
+# centos5 does not have java8 images, so supporting java 7 too.
+# unfortunately create_javah does not work in java8 correctly.
+if(ARROW_GANDIVA_JAVA7)
+ add_jar(
+ gandiva_java
+ src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java
+ src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java
+ src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java
+ src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java)
+
+ create_javah(TARGET
+ gandiva_jni_headers
+ CLASSES
+ org.apache.arrow.gandiva.evaluator.ConfigurationBuilder
+ org.apache.arrow.gandiva.evaluator.JniWrapper
+ org.apache.arrow.gandiva.evaluator.ExpressionRegistryJniHelper
+ org.apache.arrow.gandiva.exceptions.GandivaException
+ DEPENDS
+ gandiva_java
+ CLASSPATH
+ gandiva_java
+ OUTPUT_DIR
+ ${JNI_HEADERS_DIR}/jni)
+else()
+ add_jar(
+ gandiva_java
+ src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java
+ src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java
+ src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java
+ src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java
+ GENERATE_NATIVE_HEADERS
+ gandiva_jni_headers
+ DESTINATION
+ ${JNI_HEADERS_DIR}/jni)
+endif()
diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
index d333556..e5f9fe6 100644
--- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
+++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java
@@ -326,7 +326,7 @@ public class ProjectorTest extends BaseEvaluatorTest {
AtomicInteger errorCount = new AtomicInteger(0);
AtomicInteger errorCountExp = new AtomicInteger(0);
// pre-build the projector so that same projector is used for all executions.
- Projector.make(s, exprs);
+ Projector test = Projector.make(s, exprs);
IntStream.range(0, 1000).forEach(i -> {
executors.submit(() -> {
@@ -373,6 +373,7 @@ public class ProjectorTest extends BaseEvaluatorTest {
});
executors.shutdown();
executors.awaitTermination(100, java.util.concurrent.TimeUnit.SECONDS);
+ test.close();
Assert.assertEquals(errorCountExp.intValue(), errorCount.intValue());
}