You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/25 00:49:41 UTC
[arrow] branch master updated: ARROW-5054: [Release][Flight] Test
Flight in Linux/macOS release verification scripts
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6a5c43e ARROW-5054: [Release][Flight] Test Flight in Linux/macOS release verification scripts
6a5c43e is described below
commit 6a5c43ef456cae2400ab85874f0b71f7463c5afa
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Mon Jun 24 19:49:33 2019 -0500
ARROW-5054: [Release][Flight] Test Flight in Linux/macOS release verification scripts
Author: Sutou Kouhei <ko...@clear-code.com>
Author: Wes McKinney <we...@apache.org>
Closes #4613 from wesm/ARROW-5054 and squashes the following commits:
a4917cafe <Sutou Kouhei> Add Emacs configuration for editing .sh
487cf64d7 <Sutou Kouhei> Add missing ninja argument
575cfe5cd <Sutou Kouhei> Fix path
cb8ea34a4 <Sutou Kouhei> Don't create dummy top-level .git to avoid Java test failure
84bfd311c <Sutou Kouhei> Restore Go test
a5a7b766e <Sutou Kouhei> Add support GLib test without configure
e73e270cd <Sutou Kouhei> Update the required Node.js version
b81ef910c <Sutou Kouhei> Add support for verifying local source
3eaa826a5 <Sutou Kouhei> Use $PWD
49c3bda84 <Sutou Kouhei> Unify verification script again
f8e0f95ba <Wes McKinney> Fix C++ unit tests at least
21c87195e <Wes McKinney> Add test-release-verification.sh script to help
3cee0875f <Wes McKinney> Move release verification functions into a separate bash script to make reuse / testing easier
---
.dir-locals.el | 20 +++
dev/release/02-source-test.rb | 2 +
dev/release/02-source.sh | 6 +-
dev/release/post-06-csharp.sh | 2 +
dev/release/verify-release-candidate.sh | 230 +++++++++++++++++++-------------
python/setup.py | 2 +-
6 files changed, 167 insertions(+), 95 deletions(-)
diff --git a/.dir-locals.el b/.dir-locals.el
new file mode 100644
index 0000000..d0440a7
--- /dev/null
+++ b/.dir-locals.el
@@ -0,0 +1,20 @@
+;;; Licensed to the Apache Software Foundation (ASF) under one
+;;; or more contributor license agreements. See the NOTICE file
+;;; distributed with this work for additional information
+;;; regarding copyright ownership. The ASF licenses this file
+;;; to you under the Apache License, Version 2.0 (the
+;;; "License"); you may not use this file except in compliance
+;;; with the License. You may obtain a copy of the License at
+;;;
+;;; http://www.apache.org/licenses/LICENSE-2.0
+;;;
+;;; Unless required by applicable law or agreed to in writing,
+;;; software distributed under the License is distributed on an
+;;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+;;; KIND, either express or implied. See the License for the
+;;; specific language governing permissions and limitations
+;;; under the License.
+
+((sh-mode . ((indent-tabs-mode . nil)
+ (sh-indentation . 2)
+ (sh-basic-offset . 2))))
diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb
index 04daeea..e777c75 100644
--- a/dev/release/02-source-test.rb
+++ b/dev/release/02-source-test.rb
@@ -72,7 +72,9 @@ class SourceTest < Test::Unit::TestCase
def test_csharp_git_commit_information
source
Dir.chdir("#{@tag_name}/csharp") do
+ FileUtils.mv("dummy.git", "../.git")
sh("dotnet", "pack", "-c", "Release")
+ FileUtils.mv("../.git", "dummy.git")
Dir.chdir("artifacts/Apache.Arrow/Release") do
sh("unzip", "Apache.Arrow.#{@snapshot_version}.nupkg")
FileUtils.chmod(0400, "Apache.Arrow.nuspec")
diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh
index 1a34055..edaeec9 100755
--- a/dev/release/02-source.sh
+++ b/dev/release/02-source.sh
@@ -90,11 +90,13 @@ cp -R -L ${tag}.tmp ${tag}
rm -rf ${tag}.tmp
# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#.
-mkdir ${tag}/.git && cd $_
+dummy_git=${tag}/csharp/dummy.git
+mkdir ${dummy_git}
+pushd ${dummy_git}
echo ${release_hash} > HEAD
echo '[remote "origin"] url = https://github.com/apache/arrow.git' >> config
mkdir objects refs
-cd -
+popd
# Create new tarball from modified source directory
tar czf ${tarball} ${tag}
diff --git a/dev/release/post-06-csharp.sh b/dev/release/post-06-csharp.sh
index 59edc02..af61616 100755
--- a/dev/release/post-06-csharp.sh
+++ b/dev/release/post-06-csharp.sh
@@ -44,7 +44,9 @@ curl \
rm -rf ${archive_name}
tar xf ${tar_gz}
pushd ${archive_name}/csharp
+mv dummy.git ../.git
dotnet pack -c Release
+mv ../.git dummy.git
for package in artifacts/Apache.Arrow/Release/*.{nupkg,snupkg}; do
dotnet nuget push \
${package} \
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index f14c99d..d016c87 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -23,7 +23,8 @@
# - Maven >= 3.3.9
# - JDK >=7
# - gcc >= 4.8
-# - nodejs >= 6.0.0 (best way is to use nvm)
+# - Node.js >= 11.12 (best way is to use nvm)
+# - Go >= 1.11
#
# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
# LD_LIBRARY_PATH.
@@ -49,8 +50,6 @@ set -o pipefail
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
-ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
-
detect_cuda() {
if ! (which nvcc && which nvidia-smi) > /dev/null; then
return 1
@@ -60,10 +59,15 @@ detect_cuda() {
return $((${n_gpus} < 1))
}
+# Build options for the C++ library
+
if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then
ARROW_CUDA=ON
fi
: ${ARROW_CUDA:=OFF}
+: ${ARROW_FLIGHT:=ON}
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
download_dist_file() {
curl \
@@ -191,9 +195,14 @@ test_yum() {
done
}
+
setup_tempdir() {
cleanup() {
- rm -fr "$TMPDIR"
+ if [ "${TEST_SUCCESS}" = "yes" ]; then
+ rm -fr "${TMPDIR}"
+ else
+ echo "Failed to verify release candidate. See ${TMPDIR} for details."
+ fi
}
trap cleanup EXIT
TMPDIR=$(mktemp -d -t "$1.XXXXX")
@@ -208,7 +217,7 @@ setup_miniconda() {
MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
fi
- MINICONDA=`pwd`/test-miniconda
+ MINICONDA=$PWD/test-miniconda
wget -O miniconda.sh $MINICONDA_URL
bash miniconda.sh -b -p $MINICONDA
@@ -226,6 +235,17 @@ setup_miniconda() {
conda activate arrow-test
}
+# Build and test Java (Requires newer Maven -- I used 3.3.9)
+
+test_package_java() {
+ pushd java
+
+ mvn test
+ mvn package
+
+ popd
+}
+
# Build and test C++
test_and_install_cpp() {
@@ -236,6 +256,7 @@ test_and_install_cpp() {
${ARROW_CMAKE_OPTIONS}
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME
-DCMAKE_INSTALL_LIBDIR=lib
+-DARROW_FLIGHT=${ARROW_FLIGHT}
-DARROW_PLASMA=ON
-DARROW_ORC=ON
-DARROW_PYTHON=ON
@@ -249,11 +270,11 @@ ${ARROW_CMAKE_OPTIONS}
"
cmake $ARROW_CMAKE_OPTIONS ..
- make -j$NPROC
- make install
+ make -j$NPROC install
- # TODO: ARROW-5036
- ctest \
+ # TODO: ARROW-5036: plasma-serialization_tests broken
+ # TODO: ARROW-5054: libgtest.so link failure in flight-server-test
+ LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \
--exclude-regex "plasma-serialization_tests" \
-j$NPROC \
--output-on-failure \
@@ -274,6 +295,9 @@ test_python() {
if [ "${ARROW_CUDA}" = "ON" ]; then
export PYARROW_WITH_CUDA=1
fi
+ if [ "${ARROW_FLIGHT}" = "ON" ]; then
+ export PYARROW_WITH_FLIGHT=1
+ fi
python setup.py build_ext --inplace
py.test pyarrow -v --pdb
@@ -281,19 +305,22 @@ test_python() {
popd
}
-
test_glib() {
pushd c_glib
if brew --prefix libffi > /dev/null 2>&1; then
- ./configure --prefix=$ARROW_HOME \
- PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH
- else
- ./configure --prefix=$ARROW_HOME
+ PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig:$PKG_CONFIG_PATH
fi
- make -j$NPROC
- make install
+ if [ -f configure ]; then
+ ./configure --prefix=$ARROW_HOME
+ make -j$NPROC
+ make install
+ else
+ meson build --prefix=$ARROW_HOME --libdir=lib
+ ninja -C build
+ ninja -C build install
+ fi
export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH
@@ -343,10 +370,19 @@ test_ruby() {
popd
}
+test_go() {
+ pushd go/arrow
+
+ go get -v ./...
+ go test ./...
+
+ popd
+}
+
test_rust() {
# install rust toolchain in a similar fashion like test-miniconda
- export RUSTUP_HOME=`pwd`/test-rustup
- export CARGO_HOME=`pwd`/test-rustup
+ export RUSTUP_HOME=$PWD/test-rustup
+ export CARGO_HOME=$PWD/test-rustup
curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
@@ -370,85 +406,38 @@ test_rust() {
popd
}
-test_go() {
- pushd go
-
- go get -v ./...
- go test ./...
-
- popd
-}
-
-# Build and test Java (Requires newer Maven -- I used 3.3.9)
-
-test_package_java() {
- pushd java
-
- mvn test
- mvn package
-
- popd
-}
-
# Run integration tests
test_integration() {
- JAVA_DIR=`pwd`/java
- CPP_BUILD_DIR=`pwd`/cpp/build
+ JAVA_DIR=$PWD/java
+ CPP_BUILD_DIR=$PWD/cpp/build
export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
pushd integration
- python integration_test.py
-
- popd
-}
-
-setup_tempdir "arrow-$VERSION"
-echo "Working in sandbox $TMPDIR"
-cd $TMPDIR
+ INTEGRATION_TEST_ARGS=
-export ARROW_HOME=$TMPDIR/install
-export PARQUET_HOME=$TMPDIR/install
-export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
-export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
-
-if [ "$(uname)" == "Darwin" ]; then
- NPROC=$(sysctl -n hw.ncpu)
-else
- NPROC=$(nproc)
-fi
+ if [ "${ARROW_FLIGHT}" = "ON" ]; then
+ INTEGRATION_TEST_ARGS=--run_flight
+ fi
-import_gpg_keys
+ python integration_test.py $INTEGRATION_TEST_ARGS
-# By default test all functionalities.
-# To deactivate one test, deactivate the test and all of its dependents
-# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
-: ${TEST_DEFAULT:=1}
-: ${TEST_JAVA:=${TEST_DEFAULT}}
-: ${TEST_CPP:=${TEST_DEFAULT}}
-: ${TEST_GLIB:=${TEST_DEFAULT}}
-: ${TEST_RUBY:=${TEST_DEFAULT}}
-: ${TEST_PYTHON:=${TEST_DEFAULT}}
-: ${TEST_JS:=${TEST_DEFAULT}}
-: ${TEST_INTEGRATION:=${TEST_DEFAULT}}
-: ${TEST_GO:=${TEST_DEFAULT}}
-: ${TEST_RUST:=${TEST_DEFAULT}}
-: ${TEST_BINARY:=${TEST_DEFAULT}}
-: ${TEST_APT:=${TEST_DEFAULT}}
-: ${TEST_YUM:=${TEST_DEFAULT}}
+ popd
+}
-# Automatically test if its activated by a dependent
-TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
-TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION}))
-TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON}))
-TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION}))
-TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION}))
+test_source_distribution() {
+ export ARROW_HOME=$TMPDIR/install
+ export PARQUET_HOME=$TMPDIR/install
+ export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
+ export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
-if [ "$ARTIFACT" == "source" ]; then
- TARBALL=apache-arrow-$1.tar.gz
- DIST_NAME="apache-arrow-${VERSION}"
+ if [ "$(uname)" == "Darwin" ]; then
+ NPROC=$(sysctl -n hw.ncpu)
+ else
+ NPROC=$(nproc)
+ fi
git clone https://github.com/apache/arrow-testing.git
export ARROW_TEST_DATA=$PWD/arrow-testing/data
@@ -456,10 +445,6 @@ if [ "$ARTIFACT" == "source" ]; then
git clone https://github.com/apache/parquet-testing.git
export PARQUET_TEST_DATA=$PWD/parquet-testing/data
- fetch_archive $DIST_NAME
- tar xvzf ${DIST_NAME}.tar.gz
- cd ${DIST_NAME}
-
if [ ${TEST_JAVA} -gt 0 ]; then
test_package_java
fi
@@ -479,16 +464,18 @@ if [ "$ARTIFACT" == "source" ]; then
if [ ${TEST_JS} -gt 0 ]; then
test_js
fi
- if [ ${TEST_INTEGRATION} -gt 0 ]; then
- test_integration
- fi
if [ ${TEST_GO} -gt 0 ]; then
test_go
fi
if [ ${TEST_RUST} -gt 0 ]; then
test_rust
fi
-else
+ if [ ${TEST_INTEGRATION} -gt 0 ]; then
+ test_integration
+ fi
+}
+
+test_binary_distribution() {
: ${BINTRAY_REPOSITORY:=apache/arrow}
if [ ${TEST_BINARY} -gt 0 ]; then
@@ -500,7 +487,66 @@ else
if [ ${TEST_YUM} -gt 0 ]; then
test_yum
fi
+}
+
+# By default test all functionalities.
+# To deactivate one test, deactivate the test and all of its dependents
+# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
+: ${TEST_DEFAULT:=1}
+: ${TEST_SOURCE:=${TEST_DEFAULT}}
+: ${TEST_JAVA:=${TEST_DEFAULT}}
+: ${TEST_CPP:=${TEST_DEFAULT}}
+: ${TEST_GLIB:=${TEST_DEFAULT}}
+: ${TEST_RUBY:=${TEST_DEFAULT}}
+: ${TEST_PYTHON:=${TEST_DEFAULT}}
+: ${TEST_JS:=${TEST_DEFAULT}}
+: ${TEST_GO:=${TEST_DEFAULT}}
+: ${TEST_RUST:=${TEST_DEFAULT}}
+: ${TEST_INTEGRATION:=${TEST_DEFAULT}}
+: ${TEST_BINARY:=${TEST_DEFAULT}}
+: ${TEST_APT:=${TEST_DEFAULT}}
+: ${TEST_YUM:=${TEST_DEFAULT}}
+
+# Automatically test if its activated by a dependent
+TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
+TEST_PYTHON=$((${TEST_PYTHON} + ${TEST_INTEGRATION}))
+TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON}))
+TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION}))
+TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION}))
+
+: ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz}
+case "${TEST_ARCHIVE}" in
+ /*)
+ ;;
+ *)
+ TEST_ARCHIVE=${PWD}/${TEST_ARCHIVE}
+ ;;
+esac
+
+TEST_SUCCESS=no
+
+setup_tempdir "arrow-${VERSION}"
+echo "Working in sandbox ${TMPDIR}"
+cd ${TMPDIR}
+
+if [ "${ARTIFACT}" == "source" ]; then
+ dist_name="apache-arrow-${VERSION}"
+ if [ ${TEST_SOURCE} -gt 0 ]; then
+ import_gpg_keys
+ fetch_archive ${dist_name}
+ tar xf ${dist_name}.tar.gz
+ else
+ mkdir -p ${dist_name}
+ tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
+ fi
+ pushd ${dist_name}
+ test_source_distribution
+ popd
+else
+ import_gpg_keys
+ test_binary_distribution
fi
+TEST_SUCCESS=yes
echo 'Release candidate looks good!'
exit 0
diff --git a/python/setup.py b/python/setup.py
index db07d85..319d092 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -511,7 +511,7 @@ def _move_shared_libs_unix(build_prefix, build_lib, lib_name):
# If the event of not running from a git clone (e.g. from a git archive
# or a Python sdist), see if we can set the version number ourselves
default_version = '0.14.0-SNAPSHOT'
-if (not os.path.exists('../.git/index')
+if (not os.path.exists('../.git')
and not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')):
if os.path.exists('PKG-INFO'):
# We're probably in a Python sdist, setuptools_scm will handle fine