You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/17 17:46:07 UTC
[5/6] arrow git commit: ARROW-559: Add release verification script
for Linux
ARROW-559: Add release verification script for Linux
Since we're accumulating a bunch of components, I started this script which we can refine to make verifying releases easier for others.
I bootstrapped some pieces off https://github.com/apache/parquet-cpp/blob/master/dev/release/verify-release-candidate, very helpful!
This script:
* Checks GPG signature, checksums
* Installs temporary Python install for the duration of these tests
* Builds/install C++ and runs tests (with Python and Plasma)
* Builds parquet-cpp against the Arrow RC
* Python (with Parquet and Plasma extensions)
* C GLib (requires Ruby in PATH and the gems indicated in README)
* Integration tests
* JavaScript (requires NodeJS >= 6.0.0)
There are some potentially snowflake-y aspects to my environment:
* BOOST_ROOT is set to a Boost install location containing libraries built with `-fPIC`. I'm not sure what to do about this one. One maybe better option is to use system level boost and shared libraries
* Maven 3.3.9 is in PATH
* NodeJS 6.11.3 is in PATH
There are probably some other things that Linux users will run into as they run this script.
I had to compile GLib libraries in this since the ones at system level (Ubuntu 14.04) are too old.
cc @kou @xhochy
Author: Wes McKinney <we...@twosigma.com>
Closes #1102 from wesm/ARROW-559 and squashes the following commits:
8fd6530 [Wes McKinney] Use Boost shared libraries
3531927 [Wes McKinney] Add note to dev/README.md
079b5e4 [Wes McKinney] Fix comments
17f7ac0 [Wes McKinney] More fixes, finally works
adb3146 [Wes McKinney] More work on release verification script
86ef171 [Wes McKinney] Start Linux release verification script
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/59b24ba3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/59b24ba3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/59b24ba3
Branch: refs/heads/master
Commit: 59b24ba3b81bb8ead4fa74ff15ebe633d408415e
Parents: 72ad07e
Author: Wes McKinney <we...@twosigma.com>
Authored: Fri Sep 15 12:29:10 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Sep 17 13:45:20 2017 -0400
----------------------------------------------------------------------
dev/README.md | 18 ++
dev/release/verify-release-candidate.sh | 244 +++++++++++++++++++++++++++
2 files changed, 262 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/59b24ba3/dev/README.md
----------------------------------------------------------------------
diff --git a/dev/README.md b/dev/README.md
index e986abe..91d688d 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -92,3 +92,21 @@ Merge hash: 485658a5
Would you like to pick 485658a5 into another branch? (y/n):
```
For now just say n as we have 1 branch
+
+## Verifying Release Candidates
+
+We have provided a script to assist with verifying release candidates:
+
+```shell
+bash dev/release/verify-release-candidate.sh 0.7.0 0
+```
+
+Currently this only works on Linux (patches to expand to macOS welcome!). Read
+the script for information about system dependencies.
+
+On Windows, we have a script that verifies C++ and Python (requires Visual
+Studio 2015):
+
+```
+dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/59b24ba3/dev/release/verify-release-candidate.sh
----------------------------------------------------------------------
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
new file mode 100755
index 0000000..38680f4
--- /dev/null
+++ b/dev/release/verify-release-candidate.sh
@@ -0,0 +1,244 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - Ruby 2.x
+# - Plus gem dependencies, see c_glib/README
+# - Maven >= 3.3.9
+# - JDK >=7
+# - gcc >= 4.8
+# - nodejs >= 6.0.0 (best way is to use nvm)
+#
+# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
+# LD_LIBRARY_PATH
+
+case $# in
+ 2) VERSION="$1"
+ RC_NUMBER="$2"
+ ;;
+
+ *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+ exit 1
+ ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+ curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+ download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+ download_dist_file KEYS
+ gpg --import KEYS
+}
+
+fetch_archive() {
+ local dist_name=$1
+ download_rc_file ${dist_name}.tar.gz
+ download_rc_file ${dist_name}.tar.gz.asc
+ download_rc_file ${dist_name}.tar.gz.md5
+ download_rc_file ${dist_name}.tar.gz.sha512
+ gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+ gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+ sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+}
+
+setup_tempdir() {
+ cleanup() {
+ rm -fr "$TMPDIR"
+ }
+ trap cleanup EXIT
+ TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+
+setup_miniconda() {
+ # Setup short-lived miniconda for Python and integration tests
+ MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+ MINICONDA=`pwd`/test-miniconda
+
+ wget -O miniconda.sh $MINICONDA_URL
+ bash miniconda.sh -b -p $MINICONDA
+ rm -f miniconda.sh
+
+ export PATH=$MINICONDA/bin:$PATH
+
+ conda create -n arrow-test -y -q python=3.6 \
+ nomkl numpy pandas six cython
+ source activate arrow-test
+}
+
+# Build and test C++
+
+test_and_install_cpp() {
+ mkdir cpp/build
+ pushd cpp/build
+
+ cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ -DARROW_PLASMA=on \
+ -DARROW_PYTHON=on \
+ -DARROW_BOOST_USE_SHARED=on \
+ -DCMAKE_BUILD_TYPE=release \
+ -DARROW_BUILD_BENCHMARKS=on \
+ ..
+
+ make -j$NPROC
+ make install
+
+ ctest -L unittest
+ popd
+}
+
+# Build and install Parquet master so we can test the Python bindings
+
+install_parquet_cpp() {
+ git clone git@github.com:apache/parquet-cpp.git
+
+ mkdir parquet-cpp/build
+ pushd parquet-cpp/build
+
+ cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+ -DCMAKE_BUILD_TYPE=release \
+ -DPARQUET_BOOST_USE_SHARED=on \
+ -DPARQUET_BUILD_TESTS=off \
+ ..
+
+ make -j$NPROC
+ make install
+
+ popd
+}
+
+# Build and test Python
+
+test_python() {
+ pushd python
+
+ pip install -r requirements.txt
+
+ python setup.py build_ext --inplace --with-parquet --with-plasma
+ py.test pyarrow -v --pdb
+
+ popd
+}
+
+
+test_glib() {
+ # Build and test GLib, requires GLib >= 2.44 , so install that
+ # here
+ GLIB_VERSION=glib-2.53.7
+ GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz
+ curl -f -O $GLIB_URL
+ tar xf $GLIB_VERSION.tar.xz
+ pushd $GLIB_VERSION
+
+ ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME
+ make -j$NPROC
+ make install
+
+ popd
+
+ pushd c_glib
+
+ ./configure --prefix=$ARROW_HOME
+ make -j$NPROC
+ make install
+
+ NO_MAKE=yes test/run-test.sh
+
+ popd
+}
+
+test_js() {
+ pushd js
+ npm install
+ npm run validate
+ popd
+}
+
+# Build and test Java (Requires newer Maven -- I used 3.3.9)
+
+test_package_java() {
+ pushd java
+
+ mvn test
+ mvn package
+
+ popd
+}
+
+# Run integration tests
+test_integration() {
+ JAVA_DIR=`pwd`/java
+ CPP_BUILD_DIR=`pwd`/cpp/build
+
+ export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+ export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
+
+ pushd integration
+
+ python integration_test.py
+
+ popd
+}
+
+setup_tempdir "arrow-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+export ARROW_HOME=$TMPDIR/install
+export PARQUET_HOME=$TMPDIR/install
+export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
+export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
+
+NPROC=$(nproc)
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+setup_miniconda
+test_and_install_cpp
+install_parquet_cpp
+test_python
+test_glib
+test_package_java
+test_integration
+test_js
+
+echo 'Release candidate looks good!'
+exit 0