You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/13 18:02:03 UTC
arrow git commit: ARROW-816: [C++] Travis CI script cleanup,
add C++ toolchain env with Flatbuffers, RapidJSON
Repository: arrow
Updated Branches:
refs/heads/master 19da86ab9 -> 874666a61
ARROW-816: [C++] Travis CI script cleanup, add C++ toolchain env with Flatbuffers, RapidJSON
Author: Wes McKinney <we...@twosigma.com>
Closes #537 from wesm/ARROW-816 and squashes the following commits:
16992b6 [Wes McKinney] Disable Travis CI cache on OS X. brew install ccache
4621d2d [Wes McKinney] Fix variable name
dc86821 [Wes McKinney] Fixes for integration tests Travis script
5e2c226 [Wes McKinney] Change file mode
ed4be57 [Wes McKinney] Travis CI script cleanup, add C++ toolchain env with flatbuffers, rapidjson
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/874666a6
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/874666a6
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/874666a6
Branch: refs/heads/master
Commit: 874666a61c4c7bf9f1242d8bb05274b7d1bbe2bd
Parents: 19da86a
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Apr 13 14:01:56 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu Apr 13 14:01:56 2017 -0400
----------------------------------------------------------------------
.travis.yml | 1 +
ci/travis_before_script_c_glib.sh | 4 ++--
ci/travis_before_script_cpp.sh | 19 ++++++++++---------
ci/travis_env_common.sh | 31 +++++++++++++++++++++++++++++++
ci/travis_install_conda.sh | 3 +--
ci/travis_script_integration.sh | 15 ++++-----------
ci/travis_script_python.sh | 14 ++++----------
7 files changed, 53 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 4a49c71..824f62b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,6 +48,7 @@ matrix:
- compiler: clang
osx_image: xcode6.4
os: osx
+ cache:
addons:
before_script:
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_before_script_c_glib.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 1a828e7..74bdd94 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -15,14 +15,14 @@
set -ex
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
if [ $TRAVIS_OS_NAME == "osx" ]; then
brew install gtk-doc autoconf-archive gobject-introspection
fi
gem install gobject-introspection
-ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
-
pushd $ARROW_C_GLIB_DIR
: ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_before_script_cpp.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index f804a38..3f9f67c 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,19 +15,20 @@
set -ex
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+
+# Set up C++ toolchain from conda-forge packages for faster builds
+conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
if [ $TRAVIS_OS_NAME == "osx" ]; then
brew update > /dev/null
brew install jemalloc
+ brew install ccache
fi
-mkdir $CPP_BUILD_DIR
-pushd $CPP_BUILD_DIR
-
-CPP_DIR=$TRAVIS_BUILD_DIR/cpp
-
-: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
+mkdir $ARROW_CPP_BUILD_DIR
+pushd $ARROW_CPP_BUILD_DIR
CMAKE_COMMON_FLAGS="\
-DARROW_BUILD_BENCHMARKS=ON \
@@ -37,11 +38,11 @@ if [ $TRAVIS_OS_NAME == "linux" ]; then
cmake -DARROW_TEST_MEMCHECK=on \
$CMAKE_COMMON_FLAGS \
-DARROW_CXXFLAGS="-Wconversion -Werror" \
- $CPP_DIR
+ $ARROW_CPP_DIR
else
cmake $CMAKE_COMMON_FLAGS \
-DARROW_CXXFLAGS=-Werror \
- $CPP_DIR
+ $ARROW_CPP_DIR
fi
make -j4
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_env_common.sh
----------------------------------------------------------------------
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
new file mode 100755
index 0000000..5593f00
--- /dev/null
+++ b/ci/travis_env_common.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+export MINICONDA=$HOME/miniconda
+export PATH="$MINICONDA/bin:$PATH"
+export CONDA_PKGS_DIRS=$HOME/.conda_packages
+
+# C++ toolchain
+export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+
+export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
+export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+export ARROW_JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
+
+export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
+export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_install_conda.sh
----------------------------------------------------------------------
diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh
index c036e92..7d185ee 100644
--- a/ci/travis_install_conda.sh
+++ b/ci/travis_install_conda.sh
@@ -22,8 +22,7 @@ fi
wget -O miniconda.sh $MINICONDA_URL
-export MINICONDA=$HOME/miniconda
-export CONDA_PKGS_DIRS=$HOME/.conda_packages
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
mkdir -p $CONDA_PKGS_DIRS
bash miniconda.sh -b -p $MINICONDA
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_script_integration.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh
index 8ddd89b..56f5ab7 100755
--- a/ci/travis_script_integration.sh
+++ b/ci/travis_script_integration.sh
@@ -14,23 +14,16 @@
set -e
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-JAVA_DIR=${TRAVIS_BUILD_DIR}/java
-
-pushd $JAVA_DIR
+pushd $ARROW_JAVA_DIR
mvn package
popd
-pushd $TRAVIS_BUILD_DIR/integration
-
-export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/debug
-
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+pushd $ARROW_INTEGRATION_DIR
+export ARROW_CPP_EXE_PATH=$ARROW_CPP_BUILD_DIR/debug
CONDA_ENV_NAME=arrow-integration-test
conda create -y -q -n $CONDA_ENV_NAME python=3.5
http://git-wip-us.apache.org/repos/asf/arrow/blob/874666a6/ci/travis_script_python.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 549fe11..bde1fd7 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -14,17 +14,11 @@
set -e
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-PYTHON_DIR=$TRAVIS_BUILD_DIR/python
-
-# Re-use conda installation from C++
-export MINICONDA=$HOME/miniconda
-export PATH="$MINICONDA/bin:$PATH"
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
export ARROW_HOME=$ARROW_CPP_INSTALL
-pushd $PYTHON_DIR
+pushd $ARROW_PYTHON_DIR
export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
build_parquet_cpp() {
@@ -101,10 +95,10 @@ python_version_tests() {
which python
# faster builds, please
- conda install -y nomkl
+ conda install -y -q nomkl
# Expensive dependencies install from Continuum package repo
- conda install -y pip numpy pandas cython
+ conda install -y -q pip numpy pandas cython
# Build C++ libraries
build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME