You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/08/03 20:17:55 UTC
[arrow-dist] branch master updated: ARROW-1270: [Python] Build
scripts for creating macOS Python wheels in Travis CI (#2)
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-dist.git
The following commit(s) were added to refs/heads/master by this push:
new 91200bd ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis CI (#2)
91200bd is described below
commit 91200bd6aa8ed2b6ba53bd4c1abf13c7053b0269
Author: Wes McKinney <we...@users.noreply.github.com>
AuthorDate: Thu Aug 3 16:17:53 2017 -0400
ARROW-1270: [Python] Build scripts for creating macOS Python wheels in Travis CI (#2)
* Build scripts for creating macOS Python wheels in Travis CI
Contributed by Uwe L Korn, Wes McKinney
* Move arrow submodule to root
* Add Docker manylinux1 build to matrix also
* Remove errant build step
* Send all wheels to a central BinTray repo, wait for ASF INFRA
* Deploy to BinTray from all branches
---
.gitignore | 106 +++++++++++++++++++++++++++++++++++++++++++++++
.gitmodules | 6 +++
.travis.yml | 107 ++++++++++++++++++++++++++++++++----------------
README.md | 19 +++++++++
appveyor.yml | 2 +-
arrow | 1 +
descriptor.json | 27 ++++++++++++
python-macos/README.md | 20 +++++++++
python-macos/config.sh | 77 ++++++++++++++++++++++++++++++++++
python-macos/multibuild | 1 +
10 files changed, 330 insertions(+), 36 deletions(-)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..044d762
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..ade0454
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "multibuild"]
+ path = python-macos/multibuild
+ url = https://github.com/matthew-brett/multibuild.git
+[submodule "arrow"]
+ path = arrow
+ url = https://github.com/apache/arrow.git
diff --git a/.travis.yml b/.travis.yml
index 22a640d..dad8067 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,38 +1,75 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
+env:
+ global:
+ - PLAT=x86_64
+ - BUILD_COMMIT=e9f76e125b836d0fdc0a533e2fee3fca8bf4c1a1
+language: python
+python: 3.5
sudo: required
dist: trusty
-addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - gcc-4.9 # Needed for C++11
- - g++-4.9 # Needed for C++11
- - gdb
- - ccache
- - valgrind
- - libboost-dev
- - libboost-filesystem-dev
- - libboost-system-dev
- - libjemalloc-dev
- - gtk-doc-tools
- - autoconf-archive
- - libgirepository1.0-dev
-
+services:
+- docker
matrix:
+ exclude:
+ - python: 3.5
+ include:
+ - os: osx
+ language: objective-c
+ env:
+ - MB_PYTHON_VERSION=2.7
+ - NP_BUILD_DEP=1.10.1
+ - NP_TEST_DEP=1.10.1
+ - PANDAS_DEP="0.20.2"
+ - os: osx
+ language: objective-c
+ env:
+ - MB_PYTHON_VERSION=3.5
+ - NP_BUILD_DEP="1.10.1"
+ - NP_TEST_DEP="1.10.1"
+ - PANDAS_DEP="0.20.2"
+ - os: osx
+ language: objective-c
+ env:
+ - MB_PYTHON_VERSION=3.6
+ - NP_BUILD_DEP="1.11.3"
+ - NP_TEST_DEP="1.11.3"
+ - PANDAS_DEP="0.20.2"
+ - language: cpp
+ os: linux
+ before_script:
+ - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:latest
+ before_install:
+ install:
+ script:
+ - pushd arrow/python/manylinux1
+ - git clone ../../ arrow
+ - docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 .
+ - docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64
+ /io/build_arrow.sh
+before_install:
+- brew uninstall boost
+- brew update
+- brew upgrade cmake
+- BUILD_DEPENDS="numpy==$NP_BUILD_DEP six cython"
+- TEST_DEPENDS="numpy==$NP_TEST_DEP six pandas==$PANDAS_DEP"
+- pushd python-macos
+- source multibuild/common_utils.sh
+- source multibuild/travis_steps.sh
+- before_install
+- popd
+install:
+- clean_code arrow $BUILD_COMMIT
+- build_wheel arrow $PLAT
+script:
+- echo "SCRIPT"
+- pwd
+after_success:
+- echo "After success"
+- pwd
+deploy:
+ provider: bintray
+ file: descriptor.json
+ user: wesm
+ on:
+ all_branches: true
+ key:
+ secure: nuErTGtf9e3itkAe4gODHMG7jxjW8PiDargOP4df5DntE9c3TtfV7lzIRc0YmARDLPbe4tKbBd9+Sv6deimAd6ZFWYhQALVKWEANVK+ROeH6rusq60rdxiDpdtWqTF+yXqcYk76J1ifdtPtFW1wHkXfEcnmQhP93PijIiT0+oiXN4BjdReSr0FKqDqc3ZlNu6dwx6Bb0P/IJkvAKb0V6cQwfWn2vRuQj2YcOwGPHDUDyH2/b5lCR+3egLu5DiwEvGlXsUtOIMmWoBmkoTAUkE2SO9ZToZHW9+xCIzOlQYqm35t+aknItmcmbGMfdGoo4ImDdLzo4wjFSt+sjRIKvC90ondtom6ZKc3Z+KJFzuAfmnZMT23RjgcohaDoh8g269tew/5dZdQRO5GJrtV/85Ou6a3ka+rOU1bdxQ0OcOJRWsDKm+gqWRwpikUU29+a4sREKhLp8wdOocmTSlL2lcysDppogVAqv [...]
diff --git a/README.md b/README.md
index dc3c459..df7e50b 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,20 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
## Build scripts and Packaging Automation for Apache Arrow
\ No newline at end of file
diff --git a/appveyor.yml b/appveyor.yml
index c8f7c4d..48db293 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -55,7 +55,7 @@ deploy:
api_key:
secure: T6rUTOWbyxuculsGAsKosSLEdQnY+B6Vlp7bWXdA7zLYgBFfJ7sA73JOigjA6jW+
subject: wesm
- repo: pyarrow-windows-wheels
+ repo: apache-arrow-test
package: pyarrow
version: "%pyarrow_version%"
publish: true
diff --git a/arrow b/arrow
new file mode 160000
index 0000000..21a0191
--- /dev/null
+++ b/arrow
@@ -0,0 +1 @@
+Subproject commit 21a0191579e1065212d2a435a51d9721dd33c176
diff --git a/descriptor.json b/descriptor.json
new file mode 100644
index 0000000..8e9e621
--- /dev/null
+++ b/descriptor.json
@@ -0,0 +1,27 @@
+{
+ "package": {
+ "name": "pyarrow",
+ "repo": "apache-arrow-test",
+ "subject": "wesm"
+ },
+ "version": {
+ "name": "0.5.1.dev1"
+ },
+ "files": [
+ {
+ "includePattern": "arrow/python/dist/(.*\\.whl)",
+ "uploadPattern": "wheels/$1",
+ "matrixParams": {
+ "override": 1
+ }
+ },
+ {
+ "includePattern": "arrow/python/manylinux1/dist/(.*\\.whl)",
+ "uploadPattern": "wheels/$1",
+ "matrixParams": {
+ "override": 1
+ }
+ }
+ ],
+ "publish": true
+}
diff --git a/python-macos/README.md b/python-macos/README.md
new file mode 100644
index 0000000..1bc974f
--- /dev/null
+++ b/python-macos/README.md
@@ -0,0 +1,20 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# MacOS wheels for pyarrow
\ No newline at end of file
diff --git a/python-macos/config.sh b/python-macos/config.sh
new file mode 100755
index 0000000..a723788
--- /dev/null
+++ b/python-macos/config.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+function build_wheel {
+ echo `pwd`
+ echo CFLAGS=${CFLAGS}
+ echo CXXFLAGS=${CXXFLAGS}
+ echo LDFLAGS=${LDFLAGS}
+ pushd $1
+
+ wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O boost_1_60_0.tar.gz
+ tar xf boost_1_60_0.tar.gz
+ pushd boost_1_60_0
+ ./bootstrap.sh
+ ./bjam "cxxflags=-fPIC ${CFLAGS}" cflags="-fPIC ${CXXFLAGS}" --prefix=/usr/local --with-filesystem --with-date_time --with-system --with-regex install
+ popd
+
+ # Arrow is 64bit-only at the moment
+ export CFLAGS="-arch x86_64"
+ export CXXFLAGS="-arch x86_64"
+ export ARROW_HOME=/usr/local/
+ export PARQUET_HOME=/usr/local/
+ pip install cython==0.25.2 numpy==${NP_TEST_DEP}
+ pushd cpp
+ mkdir build
+ pushd build
+ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON -DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON -DMAKE=make ..
+ make -j5
+ make install
+ popd
+ popd
+
+ git clone https://github.com/apache/parquet-cpp.git
+ pushd parquet-cpp
+ mkdir build
+ pushd build
+ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF ..
+ make -j5
+ make install
+ popd
+ popd
+
+ unset ARROW_HOME
+ unset PARQUET_HOME
+ export PYARROW_WITH_PARQUET=1
+ export PYARROW_WITH_JEMALLOC=1
+ export PYARROW_BUNDLE_ARROW_CPP=1
+ export PYARROW_BUILD_TYPE='release'
+ pushd python
+ echo "python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp"
+ python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp
+ python setup.py bdist_wheel
+ ls -l dist/
+ popd
+
+ pip install delocate==0.6.4
+ delocate-wheel -L . -v python/dist/*.whl
+ popd
+}
diff --git a/python-macos/multibuild b/python-macos/multibuild
new file mode 160000
index 0000000..413205a
--- /dev/null
+++ b/python-macos/multibuild
@@ -0,0 +1 @@
+Subproject commit 413205af2f6ee6976ac5f94da3e3aa012be91762
--
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <co...@arrow.apache.org>'].