You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2018/11/06 13:59:17 UTC
[arrow] branch master updated: ARROW-3506: [Packaging] Nightly
tests for docker-compose images
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 16ffa3d ARROW-3506: [Packaging] Nightly tests for docker-compose images
16ffa3d is described below
commit 16ffa3dc4dc317fa5d67359814d4e4599a4cc719
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Tue Nov 6 14:58:46 2018 +0100
ARROW-3506: [Packaging] Nightly tests for docker-compose images
Testing at https://github.com/kszucs/crossbow/branches/all?utf8=%E2%9C%93&query=docker
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #2755 from kszucs/ARROW-3506 and squashes the following commits:
88003fd3 <Krisztián Szűcs> build arrow:cpp as well before building arrow:lint
e08e09ea <Krisztián Szűcs> fix lint commands
5f03ec43 <Krisztián Szűcs> add remaining docker tests
9ba6459d <Krisztián Szűcs> uncomment tasks from docker group
ec47011e <Krisztián Szűcs> docker test tasks
---
dev/tasks/docker-tests/travis.linux.yml | 37 +++++++
dev/tasks/tests.yml | 176 ++++++++++++++++++++++++++++++++
2 files changed, 213 insertions(+)
diff --git a/dev/tasks/docker-tests/travis.linux.yml b/dev/tasks/docker-tests/travis.linux.yml
new file mode 100644
index 0000000..9135ca5
--- /dev/null
+++ b/dev/tasks/docker-tests/travis.linux.yml
@@ -0,0 +1,37 @@
+sudo: required
+
+services:
+ - docker
+
+cache:
+ directories:
+ - $HOME/arrow_docker_cache
+
+env:
+ global:
+ - DOCKER_COMPOSE_VERSION=1.22.0
+ - ARROW_DOCKER_CACHE_DIR=$HOME/arrow_docker_cache
+{%- if environment is defined %}
+ {%- for key, value in environment.items() %}
+ - {{ key }}={{ value }}
+ {%- endfor %}
+{%- endif %}
+
+before_install:
+ - sudo rm /usr/local/bin/docker-compose
+ - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+ - chmod +x docker-compose
+ - sudo mv docker-compose /usr/local/bin
+ - docker -v
+ - docker-compose -v
+
+before_script:
+ - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow
+ - cd arrow
+ - git checkout {{ arrow.head }}
+ - git submodule update --init --recursive
+
+script:
+{%- for command in commands %}
+ - {{ command }}
+{%- endfor %}
diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml
new file mode 100644
index 0000000..f02b59f
--- /dev/null
+++ b/dev/tasks/tests.yml
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+groups:
+ # these groups are just for convenience
+ # makes it easier to submit related tasks
+ docker:
+ - docker-rust
+ - docker-cpp
+ - docker-c_glib
+ - docker-go
+ - docker-python-2.7
+ - docker-python-3.6
+ - docker-python-3.7
+ - docker-java
+ - docker-js
+ - docker-lint
+ - docker-iwyu
+ - docker-clang-format
+ - docker-hdfs-integration
+
+tasks:
+ # arbitrary_task_name:
+ # platform: osx|linux|win
+ # template: path of jinja2 templated yml
+ # params: optional extra parameters
+ # artifacts: list of regex patterns, each needs to match a single github
+ # release asset, version variable is replaced in the pattern
+ # e.g.:
+ # - pyarrow-{version}-py36(h[a-z0-9]+)_0-linux-64.tar.bz2
+
+ ############################## Language containers #########################
+
+ docker-rust:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build rust
+ - docker-compose run rust
+
+ docker-cpp:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build cpp
+ - docker-compose run cpp
+
+ docker-c_glib:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build cpp
+ - docker-compose build c_glib
+ - docker-compose run cpp
+
+ docker-go:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build go
+ - docker-compose run go
+
+ docker-js:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build js
+ - docker-compose run js
+
+ docker-java:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build java
+ - docker-compose run java
+
+ docker-python-2.7:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 2.7
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose run python
+
+ docker-python-3.6:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.6
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose run python
+
+ docker-python-3.7:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.7
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose run python
+
+ ############################## Linter tests #################################
+
+ docker-lint:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.6
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose build lint
+ - docker-compose run lint
+
+ docker-iwyu:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.6
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose build lint
+ - docker-compose run iwyu
+
+ docker-clang-format:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ environment:
+ PYTHON_VERSION: 3.6
+ commands:
+ - docker-compose build cpp
+ - docker-compose build python
+ - docker-compose build lint
+ - docker-compose run clang-format
+
+ ############################## Integration tests ############################
+
+ docker-hdfs-integration:
+ platform: linux
+ template: docker-tests/travis.linux.yml
+ params:
+ commands:
+ - docker-compose build hdfs-integration
+ - docker-compose run hdfs-integration