You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/07 17:01:53 UTC
arrow git commit: ARROW-1430: [Python] Python CI build outside of a
bash function scope, enable flake8 to fail build
Repository: arrow
Updated Branches:
refs/heads/master b0b125fd7 -> 3f2fa0381
ARROW-1430: [Python] Python CI build outside of a bash function scope, enable flake8 to fail build
This way `set -e` will have effect on any failures during the Python build. This should result in a failing build from flake8 warnings, which I will then fix
Author: Wes McKinney <we...@twosigma.com>
Closes #1059 from wesm/ARROW-1430 and squashes the following commits:
33e9d6e [Wes McKinney] Also suppress E999 in Cython linting
ffbee54 [Wes McKinney] Do not lint Cython in Python 2.7
3cf5d25 [Wes McKinney] fix another warning
4149705 [Wes McKinney] Fix flake8 warnings
06b44b6 [Wes McKinney] Move flake8 checks to correct place
b26ddff [Wes McKinney] Remove unused lines
bbc3aeb [Wes McKinney] Split parquet-cpp install into separate script, invoke travis_script_python twice
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/3f2fa038
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/3f2fa038
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/3f2fa038
Branch: refs/heads/master
Commit: 3f2fa0381191e55e50d742bec37af0814c390b94
Parents: b0b125f
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Sep 7 13:01:47 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu Sep 7 13:01:47 2017 -0400
----------------------------------------------------------------------
.travis.yml | 8 +-
ci/travis_build_parquet_cpp.sh | 50 +++++++++
ci/travis_env_common.sh | 2 +
ci/travis_script_python.sh | 138 +++++++++---------------
python/.flake8.cython | 2 +-
python/pyarrow/includes/libarrow.pxd | 2 +-
python/pyarrow/tests/test_serialization.py | 5 +-
python/pyarrow/tests/test_table.py | 6 +-
8 files changed, 114 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 016d111..c604fd2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,7 +61,9 @@ matrix:
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+ - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+ - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+ - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
- compiler: clang
osx_image: xcode6.4
os: osx
@@ -73,7 +75,9 @@ matrix:
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
+ - $TRAVIS_BUILD_DIR/ci/travis_build_parquet_cpp.sh
+ - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7
+ - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6
- language: cpp
before_script:
- docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_build_parquet_cpp.sh
----------------------------------------------------------------------
diff --git a/ci/travis_build_parquet_cpp.sh b/ci/travis_build_parquet_cpp.sh
new file mode 100755
index 0000000..4330a31
--- /dev/null
+++ b/ci/travis_build_parquet_cpp.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
+export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
+
+# $CPP_TOOLCHAIN set up in before_script_cpp
+export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+
+PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
+mkdir -p $PARQUET_DIR
+
+git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+
+pushd $PARQUET_DIR
+mkdir build-dir
+cd build-dir
+
+cmake \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=debug \
+ -DCMAKE_INSTALL_PREFIX=$ARROW_PYTHON_PARQUET_HOME \
+ -DPARQUET_BOOST_USE_SHARED=off \
+ -DPARQUET_BUILD_BENCHMARKS=off \
+ -DPARQUET_BUILD_EXECUTABLES=off \
+ -DPARQUET_BUILD_TESTS=off \
+ ..
+
+ninja
+ninja install
+
+popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_env_common.sh
----------------------------------------------------------------------
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index ff49cdf..52c7da4 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -34,6 +34,8 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
+export ARROW_PYTHON_PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+
export CMAKE_EXPORT_COMPILE_COMMANDS=1
if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/ci/travis_script_python.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index b5ba136..e2cf254 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -21,114 +21,72 @@ set -e
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
export ARROW_HOME=$ARROW_CPP_INSTALL
-export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+export PARQUET_HOME=$ARROW_PYTHON_PARQUET_HOME
export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
export PYARROW_CXXFLAGS="-Werror"
-build_parquet_cpp() {
- export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
+PYTHON_VERSION=$1
+CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
- # $CPP_TOOLCHAIN set up in before_script_cpp
- export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
+source activate $CONDA_ENV_DIR
- PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
- mkdir -p $PARQUET_DIR
+python --version
+which python
- git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+# faster builds, please
+conda install -y -q nomkl
- pushd $PARQUET_DIR
- mkdir build-dir
- cd build-dir
+# Expensive dependencies install from Continuum package repo
+conda install -y -q pip numpy pandas cython flake8
- cmake \
- -GNinja \
- -DCMAKE_BUILD_TYPE=debug \
- -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
- -DPARQUET_BOOST_USE_SHARED=off \
- -DPARQUET_BUILD_BENCHMARKS=off \
- -DPARQUET_BUILD_EXECUTABLES=off \
- -DPARQUET_BUILD_TESTS=off \
- ..
+# Build C++ libraries
+pushd $ARROW_CPP_BUILD_DIR
- ninja
- ninja install
+# Clear out prior build files
+rm -rf *
- popd
-}
-
-build_parquet_cpp
-
-function rebuild_arrow_libraries() {
- pushd $ARROW_CPP_BUILD_DIR
-
- # Clear out prior build files
- rm -rf *
-
- cmake -GNinja \
- -DARROW_BUILD_TESTS=off \
- -DARROW_BUILD_UTILITIES=off \
- -DARROW_PLASMA=on \
- -DARROW_PYTHON=on \
- -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
- $ARROW_CPP_DIR
-
- ninja
- ninja install
+cmake -GNinja \
+ -DARROW_BUILD_TESTS=off \
+ -DARROW_BUILD_UTILITIES=off \
+ -DARROW_PLASMA=on \
+ -DARROW_PYTHON=on \
+ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ $ARROW_CPP_DIR
- popd
-}
+ninja
+ninja install
-python_version_tests() {
- PYTHON_VERSION=$1
- CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+popd
- conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
- source activate $CONDA_ENV_DIR
+# Other stuff pip install
+pushd $ARROW_PYTHON_DIR
- python --version
- which python
+# Fail fast on style checks
+flake8 --count pyarrow
- # faster builds, please
- conda install -y -q nomkl
+# Check Cython files with some checks turned off
+flake8 --count --config=.flake8.cython pyarrow
- # Expensive dependencies install from Continuum package repo
- conda install -y -q pip numpy pandas cython flake8
+pip install -r requirements.txt
+python setup.py build_ext --with-parquet --with-plasma \
+ install --single-version-externally-managed --record=record.text
+popd
- # Fail fast on style checks
- flake8 --count pyarrow
+python -c "import pyarrow.parquet"
+python -c "import pyarrow.plasma"
- # Check Cython files with some checks turned off
- flake8 --count --config=.flake8.cython pyarrow
+if [ $TRAVIS_OS_NAME == "linux" ]; then
+ export PLASMA_VALGRIND=1
+fi
- # Build C++ libraries
- rebuild_arrow_libraries
+PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
- # Other stuff pip install
- pushd $ARROW_PYTHON_DIR
- pip install -r requirements.txt
- python setup.py build_ext --with-parquet --with-plasma \
- install --single-version-externally-managed --record=record.text
+if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
+ # Build documentation once
+ pushd $ARROW_PYTHON_DIR/doc
+ conda install -y -q --file=requirements.txt
+ sphinx-build -b html -d _build/doctrees -W source _build/html
popd
-
- python -c "import pyarrow.parquet"
- python -c "import pyarrow.plasma"
-
- if [ $TRAVIS_OS_NAME == "linux" ]; then
- export PLASMA_VALGRIND=1
- fi
-
- PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
- python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
-
- if [ "$PYTHON_VERSION" == "3.6" ] && [ $TRAVIS_OS_NAME == "linux" ]; then
- # Build documentation once
- pushd $ARROW_PYTHON_DIR/doc
- conda install -y -q --file=requirements.txt
- sphinx-build -b html -d _build/doctrees -W source _build/html
- popd
- fi
-}
-
-# run tests for python 2.7 and 3.6
-python_version_tests 2.7
-python_version_tests 3.6
+fi
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/.flake8.cython
----------------------------------------------------------------------
diff --git a/python/.flake8.cython b/python/.flake8.cython
index 53e4132..9255bda 100644
--- a/python/.flake8.cython
+++ b/python/.flake8.cython
@@ -17,4 +17,4 @@
[flake8]
filename = *.pyx,*.pxd,*.pxi
-ignore = E211,E901,E225,E226,E227
+ignore = E211,E901,E999,E225,E226,E227
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/includes/libarrow.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 756b213..cc35684 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -243,7 +243,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
shared_ptr[CSchema] RemoveMetadata()
cdef cppclass PrettyPrintOptions:
- int indent;
+ int indent
CStatus PrettyPrint(const CSchema& schema,
const PrettyPrintOptions& options,
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/tests/test_serialization.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index 27243b0..aca3848 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -77,8 +77,9 @@ def assert_equal(obj1, obj2):
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
else:
- assert obj1 == obj2, "Objects {} and {} are different.".format(obj1,
- obj2)
+ assert obj1 == obj2, ("Objects {} and {} are different."
+ .format(obj1, obj2))
+
PRIMITIVE_OBJECTS = [
0, 0.0, 0.9, 1 << 62, 1 << 100, 1 << 999,
http://git-wip-us.apache.org/repos/asf/arrow/blob/3f2fa038/python/pyarrow/tests/test_table.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py
index 424e518..4d5cb36 100644
--- a/python/pyarrow/tests/test_table.py
+++ b/python/pyarrow/tests/test_table.py
@@ -135,9 +135,9 @@ def test_recordbatch_from_to_pandas():
data = pd.DataFrame({
'c1': np.array([1, 2, 3, 4, 5], dtype='int64'),
'c2': np.array([1, 2, 3, 4, 5], dtype='uint32'),
- 'c2': np.random.randn(5),
- 'c3': ['foo', 'bar', None, 'baz', 'qux'],
- 'c4': [False, True, False, True, False]
+ 'c3': np.random.randn(5),
+ 'c4': ['foo', 'bar', None, 'baz', 'qux'],
+ 'c5': [False, True, False, True, False]
})
batch = pa.RecordBatch.from_pandas(data)