You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/04/08 09:52:39 UTC
[arrow] branch master updated: ARROW-4356: [CI] Add integration
(docker) test for turbodbc
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 71c0554 ARROW-4356: [CI] Add integration (docker) test for turbodbc
71c0554 is described below
commit 71c0554adade16468d8bc79df1c3dffa86cf87bf
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Mon Apr 8 11:52:24 2019 +0200
ARROW-4356: [CI] Add integration (docker) test for turbodbc
This also fixes https://issues.apache.org/jira/browse/ARROW-4985
Author: Krisztián Szűcs <sz...@gmail.com>
Author: Korn, Uwe <Uw...@blue-yonder.com>
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #4004 from xhochy/ARROW-4356 and squashes the following commits:
eede9958c <Krisztián Szűcs> add to nightlies
7199f186a <Krisztián Szűcs> add turbodbc-integration to Makefile.docker
94d6b98b3 <Uwe L. Korn> Add postgres tests
b3e70b366 <Korn, Uwe> ARROW-4356: Add integration (docker) test for turbodbc
---
Makefile.docker | 3 ++-
dev/tasks/tests.yml | 14 +++++++++++++
docker-compose.yml | 13 ++++++++++++
integration/turbodbc/Dockerfile | 38 ++++++++++++++++++++++++++++++++++
integration/turbodbc/runtest.sh | 46 +++++++++++++++++++++++++++++++++++++++++
5 files changed, 113 insertions(+), 1 deletion(-)
diff --git a/Makefile.docker b/Makefile.docker
index 01949fe..e6872f5 100644
--- a/Makefile.docker
+++ b/Makefile.docker
@@ -28,7 +28,8 @@ TESTS = dask hdfs-integration spark-integration python-nopandas
DEPENDS_ON_CPP = build-c_glib build-python build-r
DEPENDS_ON_CPP_ALPINE = build-python-alpine
DEPENDS_ON_PYTHON = build-lint build-docs build-dask build-hdfs-integration \
- build-spark-integration build-python-nopandas
+ build-spark-integration build-python-nopandas \
+ build-turbodbc-integration
DEPENDS_ON_LINT = build-iwyu build-clang-format
SERVICES = $(LANGUAGES) $(MISC) $(TESTS)
diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml
index a316381..f94fe6e 100644
--- a/dev/tasks/tests.yml
+++ b/dev/tasks/tests.yml
@@ -43,12 +43,14 @@ groups:
- docker-dask-integration
- docker-hdfs-integration
- docker-spark-integration
+ - docker-turbodbc-integration
integration:
- docker-pandas-master
- docker-dask-integration
- docker-hdfs-integration
- docker-spark-integration
+ - docker-turbodbc-integration
cpp-python:
- docker-cpp
@@ -317,6 +319,18 @@ tasks:
- docker-compose build spark-integration
- docker-compose run spark-integration
+ docker-turbodbc-integration:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.6
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose build turbodbc-integration
+ - docker-compose run turbodbc-integration
+
docker-pandas-master:
platform: linux
template: docker-tests/travis.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index e9dcd23..361eeb9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -477,6 +477,19 @@ services:
dockerfile: integration/dask/Dockerfile
volumes: *ubuntu-volumes
+ turbodbc-integration:
+ # Usage:
+ # export PYTHON_VERSION=3.6
+ # docker-compose build cpp
+ # docker-compose build python
+ # docker-compose build turbodbc-integration
+ # docker-compose run turbodbc-integration
+ image: arrow:turbodbc-integration
+ build:
+ context: .
+ dockerfile: integration/turbodbc/Dockerfile
+ volumes: *ubuntu-volumes
+
spark-integration:
# Usage:
# export PYTHON_VERSION=3.6
diff --git a/integration/turbodbc/Dockerfile b/integration/turbodbc/Dockerfile
new file mode 100644
index 0000000..26df8ad
--- /dev/null
+++ b/integration/turbodbc/Dockerfile
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM arrow:python-3.6
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ odbc-postgresql \
+ postgresql \
+ sudo \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# install turbodbc dependencies from conda-forge
+RUN conda install -c conda-forge pybind11 pytest pytest-cov mock unixodbc && \
+ conda clean --all
+
+ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json"
+
+# build and test
+CMD ["/bin/bash", "-c", "arrow/ci/docker_build_cpp.sh && \
+ arrow/ci/docker_build_python.sh && \
+ arrow/integration/turbodbc/runtest.sh"]
diff --git a/integration/turbodbc/runtest.sh b/integration/turbodbc/runtest.sh
new file mode 100755
index 0000000..31f9243
--- /dev/null
+++ b/integration/turbodbc/runtest.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# check that optional pyarrow modules are available
+# because pytest would just skip the pyarrow tests
+python -c "import pyarrow.orc"
+python -c "import pyarrow.parquet"
+
+pushd /tmp
+git clone https://github.com/xhochy/turbodbc.git
+pushd turbodbc
+git checkout arrow-0.13.0-prep
+git submodule update --init --recursive
+
+service postgresql start
+sudo -u postgres psql -U postgres -c 'CREATE DATABASE test_db;'
+sudo -u postgres psql -U postgres -c 'ALTER USER postgres WITH PASSWORD '\''password'\'';'
+export ODBCSYSINI="$(pwd)/travis/odbc/"
+
+mkdir build
+pushd build
+cmake -DCMAKE_INSTALL_PREFIX=./dist -DPYTHON_EXECUTABLE=`which python` -GNinja ..
+ninja install
+
+# TODO(ARROW-5074)
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib"
+
+ctest --output-on-failure