You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/06/21 09:26:42 UTC
[arrow] branch master updated: ARROW-5664: [Crossbow] Execute
nightly crossbow tests on CircleCI instead of Travis
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b9926b4 ARROW-5664: [Crossbow] Execute nightly crossbow tests on CircleCI instead of Travis
b9926b4 is described below
commit b9926b4a7526e8b85758b976063669163371851d
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Fri Jun 21 11:26:32 2019 +0200
ARROW-5664: [Crossbow] Execute nightly crossbow tests on CircleCI instead of Travis
The spark integration test has hit the 50mins maximum build time on travis, whereas Circle can run jobs up to 5 hours. Now it's possible to run crossbow tasks on circleci.
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #4631 from kszucs/crossbow-circleci and squashes the following commits:
c5edebc25 <Krisztián Szűcs> update readme; change branch naming order; add missing ci key to centos-7 build
d2b494454 <Krisztián Szűcs> unflatten within create_branch
0c0001063 <Krisztián Szűcs> run integration tests on circleci
---
dev/tasks/README.md | 10 ++-
dev/tasks/crossbow.py | 108 +++++++++++++++++++++++++-------
dev/tasks/docker-tests/circle.linux.yml | 41 ++++++++++++
dev/tasks/tasks.yml | 28 +++++++++
dev/tasks/tests.yml | 75 ++++++++++++++--------
5 files changed, 214 insertions(+), 48 deletions(-)
diff --git a/dev/tasks/README.md b/dev/tasks/README.md
index 619d8a2..b68ae39 100644
--- a/dev/tasks/README.md
+++ b/dev/tasks/README.md
@@ -58,8 +58,10 @@ submission. The tasks are defined in `tasks.yml`
1. [Create the queue
repository](https://help.github.com/articles/creating-a-new-repository)
-2. Enable [TravisCI](https://travis-ci.org/getting_started) and
- [Appveyor](https://www.appveyor.com/docs/) integrations on it
+2. Enable [TravisCI](https://travis-ci.org/getting_started),
+ [Appveyor](https://www.appveyor.com/docs/) and
+ [CircleCI](https://circleci.com/docs/2.0/getting-started/)
+ integrations on it
- turn off Travis' [auto cancellation](https://docs.travis-ci.com/user/customizing-the-build/#Building-only-the-latest-commit) feature on branches
@@ -92,6 +94,10 @@ submission. The tasks are defined in `tasks.yml`
- TravisCI: `https://travis-ci.org/<ghuser>/<ghrepo>/settings`
- Appveyor: `https://ci.appveyor.com/project/<ghuser>/<ghrepo>/settings/environment`
+ - CircleCI: `https://circleci.com/gh/<ghuser>/<ghrepo>/edit#env-vars`
+
+ On Appveyor check the `skip branches without appveyor.yml` checkbox on the
+ web UI under crossbow repository's settings.
7. Install Python 3.6:
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index e714188..b58ae9d 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -53,6 +53,62 @@ def md(template, *args, **kwargs):
return template.format(*map(escape, args), **toolz.valmap(escape, kwargs))
+def unflatten(mapping):
+ result = {}
+ for path, value in mapping.items():
+ parents, leaf = path[:-1], path[-1]
+ # create the hierarchy until we reach the leaf value
+ temp = result
+ for parent in parents:
+ temp.setdefault(parent, {})
+ temp = temp[parent]
+ # set the leaf value
+ temp[leaf] = value
+
+ return result
+
+
+# configurations for setting up branch skipping
+# - appveyor has a feature to skip builds without an appveyor.yml
+# - travis reads from the master branch and applies the rules
+# - circle requires the configuration to be present on all branch, even ones
+# that are configured to be skipped
+
+_default_travis_yml = """
+branches:
+ only:
+ - master
+ - /.*-travis-.*/
+
+os: linux
+dist: trusty
+language: generic
+"""
+
+_default_circle_yml = """
+version: 2
+
+jobs:
+ build:
+ machine: true
+
+workflows:
+ version: 2
+ build:
+ jobs:
+ - build:
+ filters:
+ branches:
+ only:
+ - /.*-circle-.*/
+"""
+
+_default_tree = {
+ '.travis.yml': _default_travis_yml,
+ '.circleci/config.yml': _default_circle_yml
+}
+
+
class JiraChangelog:
def __init__(self, version, username, password,
@@ -263,17 +319,28 @@ class Repo:
return pygit2.Signature(self.user_name, self.user_email,
int(time.time()))
- def create_branch(self, branch_name, files, parents=[], message='',
- signature=None):
- # 1. create tree
+ def create_tree(self, files):
builder = self.repo.TreeBuilder()
for filename, content in files.items():
- # insert the file and creating the new filetree
- blob_id = self.repo.create_blob(content)
- builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB)
+ if isinstance(content, dict):
+ # create a subtree
+ tree_id = self.create_tree(content)
+ builder.insert(filename, tree_id, pygit2.GIT_FILEMODE_TREE)
+ else:
+ # create a file
+ blob_id = self.repo.create_blob(content)
+ builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB)
tree_id = builder.write()
+ return tree_id
+
+ def create_branch(self, branch_name, files, parents=[], message='',
+ signature=None):
+ # 1. create tree
+ files = toolz.keymap(lambda path: tuple(path.split('/')), files)
+ files = unflatten(files)
+ tree_id = self.create_tree(files)
# 2. create commit with the tree created above
# TODO(kszucs): pass signature explicitly
@@ -349,7 +416,9 @@ class Queue(Repo):
# create tasks' branches
for task_name, task in job.tasks.items():
- task.branch = '{}-{}'.format(job.branch, task_name)
+ # adding CI's name to the end of the branch in order to use skip
+ # patterns on travis and circleci
+ task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name)
files = task.render_files(job=job, arrow=job.target)
branch = self.create_branch(task.branch, files=files)
self.create_tag(task.tag, branch.target)
@@ -465,8 +534,10 @@ class Task:
submitting the job to a queue.
"""
- def __init__(self, platform, template, artifacts=None, params=None):
+ def __init__(self, platform, ci, template, artifacts=None, params=None):
assert platform in {'win', 'osx', 'linux'}
+ assert ci in {'circle', 'travis', 'appveyor'}
+ self.ci = ci
self.platform = platform
self.template = template
self.artifacts = artifacts or []
@@ -479,25 +550,20 @@ class Task:
params = toolz.merge(self.params, extra_params)
template = Template(path.read_text(), undefined=StrictUndefined)
rendered = template.render(task=self, **params)
- return {self.filename: rendered}
+ return toolz.merge(_default_tree, {self.filename: rendered})
@property
def tag(self):
return self.branch
@property
- def ci(self):
- if self.platform == 'win':
- return 'appveyor'
- else:
- return 'travis'
-
- @property
def filename(self):
- if self.ci == 'appveyor':
- return 'appveyor.yml'
- else:
- return '.travis.yml'
+ config_files = {
+ 'circle': '.circleci/config.yml',
+ 'travis': '.travis.yml',
+ 'appveyor': 'appveyor.yml'
+ }
+ return config_files[self.ci]
class Job:
@@ -518,7 +584,7 @@ class Job:
with StringIO() as buf:
yaml.dump(self, buf)
content = buf.getvalue()
- return {'job.yml': content}
+ return toolz.merge(_default_tree, {'job.yml': content})
@property
def email(self):
diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml
new file mode 100644
index 0000000..47f1817
--- /dev/null
+++ b/dev/tasks/docker-tests/circle.linux.yml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: 2
+jobs:
+ build:
+ machine:
+ image: circleci/classic:201808-01
+ steps:
+ - run: docker -v
+ - run: docker-compose -v
+ - run: git clone --no-checkout {{ arrow.remote }} arrow
+ - run: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+ - run: git -C arrow checkout {{ arrow.head }}
+ - run: git -C arrow submodule update --init --recursive
+ - run: |
+ pushd arrow
+ {%- for command in commands %}
+ {{ command }}
+ {%- endfor %}
+ popd
+
+workflows:
+ version: 2
+ build:
+ jobs:
+ - build
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 910650b..16e90f7 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -66,6 +66,7 @@ tasks:
############################## Conda Linux ##################################
conda-linux-gcc-py27:
+ ci: travis
platform: linux
template: conda-recipes/travis.linux.yml
params:
@@ -75,6 +76,7 @@ tasks:
- pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
conda-linux-gcc-py36:
+ ci: travis
platform: linux
template: conda-recipes/travis.linux.yml
params:
@@ -84,6 +86,7 @@ tasks:
- pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2
conda-linux-gcc-py37:
+ ci: travis
platform: linux
template: conda-recipes/travis.linux.yml
params:
@@ -95,6 +98,7 @@ tasks:
############################## Conda OSX ####################################
conda-osx-clang-py27:
+ ci: travis
platform: osx
template: conda-recipes/travis.osx.yml
params:
@@ -104,6 +108,7 @@ tasks:
- pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2
conda-osx-clang-py36:
+ ci: travis
platform: osx
template: conda-recipes/travis.osx.yml
params:
@@ -124,6 +129,7 @@ tasks:
############################## Conda Windows ################################
conda-win-vs2015-py36:
+ ci: appveyor
platform: win
template: conda-recipes/appveyor.yml
params:
@@ -133,6 +139,7 @@ tasks:
- pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2
conda-win-vs2015-py37:
+ ci: appveyor
platform: win
template: conda-recipes/appveyor.yml
params:
@@ -144,6 +151,7 @@ tasks:
############################## Wheel Linux ##################################
wheel-linux-cp27m:
+ ci: travis
platform: linux
template: python-wheels/travis.linux.yml
params:
@@ -154,6 +162,7 @@ tasks:
- pyarrow-{no_rc_version}-cp27-cp27m-manylinux1_x86_64.whl
wheel-linux-cp27mu:
+ ci: travis
platform: linux
template: python-wheels/travis.linux.yml
params:
@@ -165,6 +174,7 @@ tasks:
- pyarrow-{no_rc_version}-cp27-cp27mu-manylinux1_x86_64.whl
wheel-linux-cp35m:
+ ci: travis
platform: linux
template: python-wheels/travis.linux.yml
params:
@@ -176,6 +186,7 @@ tasks:
- pyarrow-{no_rc_version}-cp35-cp35m-manylinux1_x86_64.whl
wheel-linux-cp36m:
+ ci: travis
platform: linux
template: python-wheels/travis.linux.yml
params:
@@ -187,6 +198,7 @@ tasks:
- pyarrow-{no_rc_version}-cp36-cp36m-manylinux1_x86_64.whl
wheel-linux-cp37m:
+ ci: travis
platform: linux
template: python-wheels/travis.linux.yml
params:
@@ -200,6 +212,7 @@ tasks:
############################## Wheel OSX ####################################
wheel-osx-cp27m:
+ ci: travis
platform: osx
template: python-wheels/travis.osx.yml
params:
@@ -208,6 +221,7 @@ tasks:
- pyarrow-{no_rc_version}-cp27-cp27m-macosx_10_6_intel.whl
wheel-osx-cp35m:
+ ci: travis
platform: osx
template: python-wheels/travis.osx.yml
params:
@@ -216,6 +230,7 @@ tasks:
- pyarrow-{no_rc_version}-cp35-cp35m-macosx_10_6_intel.whl
wheel-osx-cp36m:
+ ci: travis
platform: osx
template: python-wheels/travis.osx.yml
params:
@@ -224,6 +239,7 @@ tasks:
- pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_6_intel.whl
wheel-osx-cp37m:
+ ci: travis
platform: osx
template: python-wheels/travis.osx.yml
params:
@@ -234,6 +250,7 @@ tasks:
############################## Wheel Windows ################################
wheel-win-cp35m:
+ ci: appveyor
platform: win
template: python-wheels/appveyor.yml
params:
@@ -242,6 +259,7 @@ tasks:
- pyarrow-{no_rc_version}-cp35-cp35m-win_amd64.whl
wheel-win-cp36m:
+ ci: appveyor
platform: win
template: python-wheels/appveyor.yml
params:
@@ -250,6 +268,7 @@ tasks:
- pyarrow-{no_rc_version}-cp36-cp36m-win_amd64.whl
wheel-win-cp37m:
+ ci: appveyor
platform: win
template: python-wheels/appveyor.yml
params:
@@ -260,6 +279,7 @@ tasks:
############################## Linux PKGS ####################################
debian-stretch:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -323,6 +343,7 @@ tasks:
# Don't use this for now. We can't complete this task in 50min on Travis CI.
debian-stretch-arm64:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.arm64.yml
params:
@@ -360,6 +381,7 @@ tasks:
- libparquet14_{no_rc_version}-1_amd64.deb
ubuntu-xenial:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -408,6 +430,7 @@ tasks:
- plasma-store-server_{no_rc_version}-1_amd64.deb
ubuntu-bionic:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -459,6 +482,7 @@ tasks:
- plasma-store-server_{no_rc_version}-1_amd64.deb
ubuntu-cosmic:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -510,6 +534,7 @@ tasks:
- plasma-store-server_{no_rc_version}-1_amd64.deb
centos-6:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -527,6 +552,7 @@ tasks:
- plasma-libs-{no_rc_version}-1.el6.x86_64.rpm
centos-7:
+ ci: travis
platform: linux
template: linux-packages/travis.linux.yml
params:
@@ -557,12 +583,14 @@ tasks:
############################## Gandiva Tasks ################################
gandiva-jar-trusty:
+ ci: travis
platform: linux
template: gandiva-jars/travis.linux.yml
artifacts:
- arrow-gandiva-{no_rc_version}-SNAPSHOT.jar
gandiva-jar-osx:
+ ci: travis
platform: osx
template: gandiva-jars/travis.osx.yml
artifacts:
diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml
index fff5a60..f8dc817 100644
--- a/dev/tasks/tests.yml
+++ b/dev/tasks/tests.yml
@@ -77,8 +77,9 @@ tasks:
############################## Language containers #########################
docker-r:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build cpp
@@ -86,40 +87,45 @@ tasks:
- docker-compose run r
docker-rust:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build rust
- docker-compose run rust
docker-cpp:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build cpp
- docker-compose run cpp
docker-cpp-alpine:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build cpp-alpine
- docker-compose run cpp-alpine
docker-cpp-cmake32:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build cpp-cmake32
- docker-compose run cpp-cmake32
docker-c_glib:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build cpp
@@ -127,32 +133,36 @@ tasks:
- docker-compose run cpp
docker-go:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build go
- docker-compose run go
docker-js:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build js
- docker-compose run js
docker-java:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
commands:
- docker-compose build java
- docker-compose run java
docker-python-2.7:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 2.7
@@ -162,8 +172,9 @@ tasks:
- docker-compose run python
docker-python-3.6:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -173,8 +184,9 @@ tasks:
- docker-compose run python
docker-python-3.7:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.7
@@ -184,8 +196,9 @@ tasks:
- docker-compose run python
docker-python-2.7-alpine:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 2.7
@@ -195,8 +208,9 @@ tasks:
- docker-compose run python-alpine
docker-python-3.6-alpine:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -206,8 +220,9 @@ tasks:
- docker-compose run python-alpine
docker-python-2.7-nopandas:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 2.7
@@ -218,8 +233,9 @@ tasks:
- docker-compose run python-nopandas
docker-python-3.6-nopandas:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -232,8 +248,9 @@ tasks:
###################### Documentation building tests #########################
docker-docs:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -246,8 +263,9 @@ tasks:
############################## Linter tests #################################
docker-lint:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -258,8 +276,9 @@ tasks:
- docker-compose run lint
docker-iwyu:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -270,8 +289,9 @@ tasks:
- docker-compose run iwyu
docker-clang-format:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -284,8 +304,9 @@ tasks:
############################## Integration tests ############################
docker-dask-integration:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -296,8 +317,9 @@ tasks:
- docker-compose run dask-integration
docker-hdfs-integration:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -308,8 +330,9 @@ tasks:
- docker-compose run hdfs-integration
docker-spark-integration:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -320,8 +343,9 @@ tasks:
- docker-compose run spark-integration
docker-turbodbc-integration:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6
@@ -332,8 +356,9 @@ tasks:
- docker-compose run turbodbc-integration
docker-pandas-master:
+ ci: circle
platform: linux
- template: docker-tests/travis.linux.yml
+ template: docker-tests/circle.linux.yml
params:
environment:
PYTHON_VERSION: 3.6