You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/28 03:27:59 UTC
[arrow] branch master updated: ARROW-5511: [Packaging] Enable
Flight in Conda packages
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a0c1c04 ARROW-5511: [Packaging] Enable Flight in Conda packages
a0c1c04 is described below
commit a0c1c04feb30a08a64c8ff73b041ed2dff205067
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Thu Jun 27 22:27:51 2019 -0500
ARROW-5511: [Packaging] Enable Flight in Conda packages
Author: Krisztián Szűcs <sz...@gmail.com>
Author: Wes McKinney <we...@apache.org>
Author: David Li <li...@gmail.com>
Closes #4473 from lihalite/conda-flight and squashes the following commits:
909b798c2 <Wes McKinney> Also build uriparser bundled in windows
55d2f414a <Wes McKinney> Build Flight in Windows conda packages
c1baa4c43 <Krisztián Szűcs> skip tests instead
c0b95ac48 <Krisztián Szűcs> debug env vars
e764a59e1 <Krisztián Szűcs> use SRC_DIR
f61250852 <Krisztián Szűcs> export ARROW_TEST_DATA on the template level
8765af149 <Krisztián Szűcs> pass ARROW_TEST_DATA
d809fdf2e <Krisztián Szűcs> bundle uriparser
e9c4cd645 <David Li> Enable Flight in Conda packages
---
dev/tasks/conda-recipes/appveyor.win.yml | 1 +
dev/tasks/conda-recipes/arrow-cpp/bld.bat | 2 ++
dev/tasks/conda-recipes/arrow-cpp/build.sh | 2 ++
dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 9 +++++++++
dev/tasks/conda-recipes/azure.linux.yml | 1 +
dev/tasks/conda-recipes/azure.osx.yml | 1 +
dev/tasks/conda-recipes/azure.win.yml | 1 +
dev/tasks/conda-recipes/pyarrow/bld.bat | 5 +++--
dev/tasks/conda-recipes/pyarrow/build.sh | 1 +
dev/tasks/conda-recipes/pyarrow/meta.yaml | 3 ++-
python/pyarrow/tests/test_flight.py | 3 +++
11 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/dev/tasks/conda-recipes/appveyor.win.yml b/dev/tasks/conda-recipes/appveyor.win.yml
index 121595a..48252f2 100644
--- a/dev/tasks/conda-recipes/appveyor.win.yml
+++ b/dev/tasks/conda-recipes/appveyor.win.yml
@@ -29,6 +29,7 @@ test_script:
- git clone --no-checkout {{ arrow.remote }} arrow || exit /B
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
- git -C arrow checkout FETCH_HEAD || exit /B
+ - git -C arrow submodule update --init --recursive || exit /B
- pushd arrow\dev\tasks\conda-recipes
diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld.bat b/dev/tasks/conda-recipes/arrow-cpp/bld.bat
index 429cb64..d3ea1b2 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/bld.bat
+++ b/dev/tasks/conda-recipes/arrow-cpp/bld.bat
@@ -10,10 +10,12 @@ cmake -G "%CMAKE_GENERATOR%" ^
-DARROW_BUILD_TESTS:BOOL=OFF ^
-DARROW_BUILD_UTILITIES:BOOL=OFF ^
-DCMAKE_BUILD_TYPE=release ^
+ -DARROW_FLIGHT:BOOL=ON ^
-DARROW_PYTHON:BOOL=ON ^
-DARROW_PARQUET:BOOL=ON ^
-DARROW_GANDIVA:BOOL=ON ^
-DARROW_ORC:BOOL=ON ^
+ -Duriparser_SOURCE=BUNDLED ^
..
cmake --build . --target INSTALL --config Release
diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh
index b18862a..0c4414d 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/build.sh
+++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh
@@ -19,12 +19,14 @@ cmake \
-DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \
-DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \
-DARROW_DEPENDENCY_SOURCE=SYSTEM \
+ -Duriparser_SOURCE=BUNDLED \
-DARROW_PACKAGE_PREFIX=$PREFIX \
-DARROW_BOOST_USE_SHARED=ON \
-DARROW_BUILD_BENCHMARKS=OFF \
-DARROW_BUILD_UTILITIES=OFF \
-DARROW_BUILD_TESTS=OFF \
-DARROW_JEMALLOC=ON \
+ -DARROW_FLIGHT=ON \
-DARROW_PLASMA=ON \
-DARROW_PYTHON=ON \
-DARROW_PARQUET=ON \
diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
index 877bfc6..3fd43f3 100644
--- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
+++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml
@@ -22,10 +22,12 @@ requirements:
host:
- boost-cpp
- brotli
+ - c-ares
- double-conversion
- flatbuffers
- gflags
- glog
+ - grpc-cpp
- libprotobuf
- clangdev 7.*
- llvmdev 7.*
@@ -44,9 +46,11 @@ requirements:
- {{ pin_compatible('numpy', lower_bound='1.14') }}
- boost-cpp
- brotli
+ - c-ares
- double-conversion
- gflags
- glog
+ - grpc-cpp
- lz4-c
- python
- re2
@@ -58,6 +62,7 @@ test:
commands:
# headers
- test -f $PREFIX/include/arrow/api.h # [unix]
+ - test -f $PREFIX/include/arrow/flight/types.h # [unix]
- test -f $PREFIX/include/plasma/client.h # [unix]
- test -f $PREFIX/include/gandiva/engine.h # [unix]
- test -f $PREFIX/include/parquet/api/reader.h # [unix]
@@ -67,17 +72,20 @@ test:
# static
- test -f $PREFIX/lib/libarrow.a # [unix]
+ - test -f $PREFIX/lib/libarrow_flight.a # [unix]
- test -f $PREFIX/lib/libarrow_python.a # [unix]
- test -f $PREFIX/lib/libplasma.a # [unix]
- test -f $PREFIX/lib/libparquet.a # [unix]
- test -f $PREFIX/lib/libgandiva.a # [unix]
- if not exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win]
+ - if not exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win]
- if not exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win]
- if not exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win]
- if not exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win]
# shared
- test -f $PREFIX/lib/libarrow.so # [linux]
+ - test -f $PREFIX/lib/libarrow_flight.so # [linux]
- test -f $PREFIX/lib/libarrow_python.so # [linux]
- test -f $PREFIX/lib/libparquet.so # [linux]
- test -f $PREFIX/lib/libgandiva.so # [linux]
@@ -88,6 +96,7 @@ test:
- test -f $PREFIX/lib/libparquet.dylib # [osx]
- test -f $PREFIX/lib/libplasma.dylib # [osx]
- if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win]
+ - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win]
- if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win]
- if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win]
- if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win]
diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml
index 355f3b1..e398dca 100755
--- a/dev/tasks/conda-recipes/azure.linux.yml
+++ b/dev/tasks/conda-recipes/azure.linux.yml
@@ -23,6 +23,7 @@ jobs:
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
+ git -C arrow submodule update --init --recursive
displayName: Clone arrow
- script: CI=azure ./run_docker_build.sh
diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml
index 7ff3570..95f76f0 100755
--- a/dev/tasks/conda-recipes/azure.osx.yml
+++ b/dev/tasks/conda-recipes/azure.osx.yml
@@ -30,6 +30,7 @@ jobs:
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
+ git -C arrow submodule update --init --recursive
displayName: Clone arrow
- script: |
diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml
index 02fc91a..6902908 100755
--- a/dev/tasks/conda-recipes/azure.win.yml
+++ b/dev/tasks/conda-recipes/azure.win.yml
@@ -52,6 +52,7 @@ jobs:
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
+ git -C arrow submodule update --init --recursive
displayName: Clone arrow
# Configure the VM
diff --git a/dev/tasks/conda-recipes/pyarrow/bld.bat b/dev/tasks/conda-recipes/pyarrow/bld.bat
index d02ceaf..28eb76a 100644
--- a/dev/tasks/conda-recipes/pyarrow/bld.bat
+++ b/dev/tasks/conda-recipes/pyarrow/bld.bat
@@ -14,10 +14,11 @@ copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\
SET ARROW_HOME=%LIBRARY_PREFIX%
SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION%
+SET PYARROW_WITH_FLIGHT=1
+SET PYARROW_WITH_GANDIVA=1
+SET PYARROW_WITH_PARQUET=1
"%PYTHON%" setup.py ^
build_ext --build-type=release ^
- --with-parquet ^
- --with-gandiva ^
install --single-version-externally-managed ^
--record=record.txt
if errorlevel 1 exit 1
diff --git a/dev/tasks/conda-recipes/pyarrow/build.sh b/dev/tasks/conda-recipes/pyarrow/build.sh
index 98b10a4..161546f 100644
--- a/dev/tasks/conda-recipes/pyarrow/build.sh
+++ b/dev/tasks/conda-recipes/pyarrow/build.sh
@@ -12,6 +12,7 @@ cd python
$PYTHON setup.py \
build_ext --build-type=release \
+ --with-flight \
--with-orc \
--with-plasma \
--with-parquet \
diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml
index 8198390..acd89af 100644
--- a/dev/tasks/conda-recipes/pyarrow/meta.yaml
+++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml
@@ -45,6 +45,7 @@ requirements:
test:
imports:
- pyarrow
+ - pyarrow.flight # [py>=35]
- pyarrow.plasma # [unix]
- pyarrow.parquet
- pyarrow.gandiva # [py>=35]
@@ -54,7 +55,7 @@ test:
- hypothesis
commands:
- - pytest --pyargs pyarrow
+ - pytest -m "not requires_testing_data" --pyargs pyarrow
about:
home: http://github.com/apache/arrow
diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py
index 3f83a1c..a8973ab 100644
--- a/python/pyarrow/tests/test_flight.py
+++ b/python/pyarrow/tests/test_flight.py
@@ -618,6 +618,7 @@ def test_location_invalid():
@pytest.mark.slow
+@pytest.mark.requires_testing_data
def test_tls_fails():
"""Make sure clients cannot connect when cert verification fails."""
certs = example_tls_certs()
@@ -635,6 +636,7 @@ def test_tls_fails():
client.do_get(flight.Ticket(b'ints'))
+@pytest.mark.requires_testing_data
def test_tls_do_get():
"""Try a simple do_get call over TLS."""
table = simple_ints_table()
@@ -650,6 +652,7 @@ def test_tls_do_get():
assert data.equals(table)
+@pytest.mark.requires_testing_data
def test_tls_override_hostname():
"""Check that incorrectly overriding the hostname fails."""
certs = example_tls_certs()