You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/17 17:07:57 UTC
[arrow] branch master updated: ARROW-5520: [Packaging][deb] Add
support for building on arm64
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6f5bebc ARROW-5520: [Packaging][deb] Add support for building on arm64
6f5bebc is described below
commit 6f5bebc3411984dcedeee6b401fb16e9120f70b1
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Mon Jun 17 12:07:49 2019 -0500
ARROW-5520: [Packaging][deb] Add support for building on arm64
We don't have NVIDIA CUDA toolkit and LLVM 7 on arm64.
This includes Crossbow build configuration for arm64 but this is not
used yet. Because building arm64 packages can't finish in 50min on
Travis CI.
Author: Sutou Kouhei <ko...@clear-code.com>
Closes #4588 from kou/packaging-deb-arm64 and squashes the following commits:
f97736d63 <Sutou Kouhei> Add support for building on arm64
---
dev/tasks/linux-packages/apt/build.sh | 9 ++-
.../apt/debian-stretch-arm64/Dockerfile | 77 ++++++++++++++++++++++
.../apt/debian-stretch-arm64/qemu-dummy-static | 33 ++++++++++
dev/tasks/linux-packages/debian/control | 40 +++++------
dev/tasks/linux-packages/debian/rules | 67 ++++++++++++-------
dev/tasks/linux-packages/package-task.rb | 2 +
.../{travis.linux.yml => travis.linux.arm64.yml} | 23 ++++++-
dev/tasks/linux-packages/travis.linux.yml | 7 +-
dev/tasks/tasks.yml | 44 ++++++++++++-
9 files changed, 246 insertions(+), 56 deletions(-)
diff --git a/dev/tasks/linux-packages/apt/build.sh b/dev/tasks/linux-packages/apt/build.sh
index b4a3fd6..a6a1cc4 100755
--- a/dev/tasks/linux-packages/apt/build.sh
+++ b/dev/tasks/linux-packages/apt/build.sh
@@ -41,7 +41,7 @@ case "${distribution}" in
component=universe
;;
esac
-specific_debian_dir="debian.${distribution}-${code_name}"
+architecture=$(dpkg-architecture -q DEB_BUILD_ARCH)
run mkdir -p build
run cp /host/tmp/${PACKAGE}-${VERSION}.tar.gz \
@@ -59,8 +59,11 @@ case "${VERSION}" in
;;
esac
run cd ${PACKAGE}-${VERSION}/
-if [ -d "/host/tmp/${specific_debian_dir}" ]; then
- run cp -rp "/host/tmp/${specific_debian_dir}" debian
+platform="${distribution}-${code_name}"
+if [ -d "/host/tmp/debian.${platform}-${architecture}" ]; then
+ run cp -rp "/host/tmp/debian.${platform}-${architecture}" debian
+elif [ -d "/host/tmp/debian.${platform}" ]; then
+ run cp -rp "/host/tmp/debian.${platform}" debian
else
run cp -rp "/host/tmp/debian" debian
fi
diff --git a/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile
new file mode 100644
index 0000000..76365a0
--- /dev/null
+++ b/dev/tasks/linux-packages/apt/debian-stretch-arm64/Dockerfile
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM arm64v8/debian:stretch
+
+COPY qemu-* /usr/bin/
+
+RUN \
+ echo "debconf debconf/frontend select Noninteractive" | \
+ debconf-set-selections
+
+ARG DEBUG
+
+RUN \
+ echo "deb http://deb.debian.org/debian stretch-backports main" > \
+ /etc/apt/sources.list.d/backports.list
+
+RUN \
+ quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
+ apt update ${quiet} && \
+ apt install -y -V ${quiet} \
+ bison \
+ build-essential \
+ cmake \
+ devscripts \
+ flex \
+ git \
+ gtk-doc-tools \
+ libboost-filesystem-dev \
+ libboost-regex-dev \
+ libboost-system-dev \
+ libbrotli-dev \
+ libc-ares-dev \
+ libdouble-conversion-dev \
+ libgirepository1.0-dev \
+ libglib2.0-doc \
+ libgoogle-glog-dev \
+ liblz4-dev \
+ libre2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libzstd-dev \
+ lsb-release \
+ ninja-build \
+ pkg-config \
+ python3-dev \
+ python3-numpy \
+ python3-pip \
+ tzdata && \
+ apt install -y -V -t stretch-backports ${quiet} \
+ debhelper \
+ libgmock-dev \
+ libgrpc++-dev \
+ libgtest-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ protobuf-compiler \
+ protobuf-compiler-grpc \
+ rapidjson-dev && \
+ pip3 install --upgrade meson && \
+ ln -s /usr/local/bin/meson /usr/bin/ && \
+ apt clean && \
+ rm -rf /var/lib/apt/lists/*
diff --git a/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static b/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static
new file mode 100755
index 0000000..82f0ea5
--- /dev/null
+++ b/dev/tasks/linux-packages/apt/debian-stretch-arm64/qemu-dummy-static
@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Do nothing. This exists only for not requiring qemu-aarch64-static copy.
+# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or
+# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian
+# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image.
+#
+# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*",
+# the "COPY" is failed. It means that we always require "qemu*" even if we
+# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file,
+# the "COPY" isn't failed. It means that we can copy "qemu*" only when we
+# need.
+#
+# See also "script" in dev/tasks/linux-packages/travis.linux.yml.
+# Travis CI uses old Ubuntu. So we need to put "qemu-aarch64-static" into
+# this directory.
diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control
index 3febdc1..69e7fa6 100644
--- a/dev/tasks/linux-packages/debian/control
+++ b/dev/tasks/linux-packages/debian/control
@@ -4,7 +4,7 @@ Priority: optional
Maintainer: Apache Arrow Developers <de...@arrow.apache.org>
Build-Depends:
bison,
- clang-7,
+ clang-7 [!arm64],
cmake,
debhelper (>= 12),
flex,
@@ -25,7 +25,7 @@ Build-Depends:
libssl-dev,
libzstd-dev,
ninja-build,
- nvidia-cuda-toolkit,
+ nvidia-cuda-toolkit [!arm64],
pkg-config,
python3-dev,
python3-numpy,
@@ -63,7 +63,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libarrow-cuda14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -112,7 +112,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libarrow-cuda-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -136,7 +136,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libgandiva14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -150,7 +150,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: libgandiva-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -163,7 +163,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: libplasma14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -176,7 +176,7 @@ Description: Plasma is an in-memory object store and cache for big data.
Package: plasma-store-server
Section: utils
-Architecture: any
+Architecture: i386 amd64
Pre-Depends: ${misc:Pre-Depends}
Depends:
${misc:Depends},
@@ -188,7 +188,7 @@ Description: Plasma is an in-memory object store and cache for big data.
Package: libplasma-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -274,7 +274,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libarrow-cuda-glib14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -288,7 +288,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: gir1.2-arrow-cuda-1.0
Section: introspection
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${gir:Depends},
@@ -299,7 +299,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libarrow-cuda-glib-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -313,7 +313,7 @@ Description: Apache Arrow is a data processing library for analysis
Package: libgandiva-glib14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -328,7 +328,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: gir1.2-gandiva-1.0
Section: introspection
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${gir:Depends},
@@ -340,7 +340,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: libgandiva-glib-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -355,7 +355,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: libgandiva-glib-doc
Section: doc
-Architecture: all
+Architecture: i386 amd64
Multi-Arch: foreign
Depends:
${misc:Depends}
@@ -367,7 +367,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
Package: libplasma-glib14
Section: libs
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Pre-Depends: ${misc:Pre-Depends}
Depends:
@@ -381,7 +381,7 @@ Description: Plasma is an in-memory object store and cache for big data.
Package: gir1.2-plasma-1.0
Section: introspection
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${gir:Depends},
@@ -392,7 +392,7 @@ Description: Plasma is an in-memory object store and cache for big data.
Package: libplasma-glib-dev
Section: libdevel
-Architecture: any
+Architecture: i386 amd64
Multi-Arch: same
Depends:
${misc:Depends},
@@ -406,7 +406,7 @@ Description: Plasma is an in-memory object store and cache for big data.
Package: libplasma-glib-doc
Section: doc
-Architecture: all
+Architecture: i386 amd64
Multi-Arch: foreign
Depends:
${misc:Depends}
diff --git a/dev/tasks/linux-packages/debian/rules b/dev/tasks/linux-packages/debian/rules
index efb0c55..32ea573 100755
--- a/dev/tasks/linux-packages/debian/rules
+++ b/dev/tasks/linux-packages/debian/rules
@@ -14,36 +14,53 @@ BUILD_TYPE=release
dh $@ --with gir
override_dh_auto_configure:
- dh_auto_configure \
- --sourcedirectory=cpp \
- --builddirectory=cpp_build \
- --buildsystem=cmake+ninja \
- -- \
- -DARROW_BUILD_TESTS=OFF \
- -DARROW_CUDA=ON \
- -DARROW_FLIGHT=ON \
- -DARROW_GANDIVA=ON \
- -DARROW_GANDIVA_JAVA=OFF \
- -DARROW_ORC=ON \
- -DARROW_PARQUET=ON \
- -DARROW_PLASMA=ON \
- -DARROW_PYTHON=ON \
- -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
- -DPythonInterp_FIND_VERSION=ON \
+ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \
+ ARROW_CUDA=ON; \
+ ARROW_PLASMA=ON; \
+ else \
+ ARROW_CUDA=OFF; \
+ ARROW_PLASMA=OFF; \
+ fi; \
+ if dpkg -l clang-7 > /dev/null 2>&1; then \
+ ARROW_GANDIVA=ON; \
+ else \
+ ARROW_GANDIVA=OFF; \
+ fi; \
+ if dpkg -l libprotobuf-dev > /dev/null 2>&1; then \
+ cpp_build_system=cmake+ninja; \
+ else \
+ cpp_build_system=cmake; \
+ fi; \
+ dh_auto_configure \
+ --sourcedirectory=cpp \
+ --builddirectory=cpp_build \
+ --buildsystem=$${cpp_build_system} \
+ -- \
+ -DARROW_BUILD_TESTS=OFF \
+ -DARROW_CUDA=$${ARROW_CUDA} \
+ -DARROW_FLIGHT=ON \
+ -DARROW_GANDIVA=$${ARROW_GANDIVA} \
+ -DARROW_GANDIVA_JAVA=OFF \
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
+ -DARROW_PLASMA=$${ARROW_GANDIVA} \
+ -DARROW_PYTHON=ON \
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+ -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
+ -DPythonInterp_FIND_VERSION=ON \
-DPythonInterp_FIND_VERSION_MAJOR=3
override_dh_auto_build:
dh_auto_build \
--sourcedirectory=cpp \
- --builddirectory=cpp_build \
- --buildsystem=cmake+ninja
- dh_auto_configure \
- --sourcedirectory=c_glib \
- --builddirectory=c_glib_build \
- --buildsystem=meson+ninja \
- -- \
- -Darrow_cpp_build_type=$(BUILD_TYPE) \
- -Darrow_cpp_build_dir=$(CURDIR)/cpp_build \
+ --builddirectory=cpp_build
+ dh_auto_configure \
+ --sourcedirectory=c_glib \
+ --builddirectory=c_glib_build \
+ --buildsystem=meson+ninja \
+ -- \
+ -Darrow_cpp_build_type=$(BUILD_TYPE) \
+ -Darrow_cpp_build_dir=$(CURDIR)/cpp_build \
-Dgtk_doc=true
env \
LD_LIBRARY_PATH=$(CURDIR)/cpp_build/$(BUILD_TYPE) \
diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb
index 3f7fc1e..d1c08df 100644
--- a/dev/tasks/linux-packages/package-task.rb
+++ b/dev/tasks/linux-packages/package-task.rb
@@ -247,6 +247,8 @@ VERSION=#{@deb_upstream_version}
if targets.empty?
targets = [
"debian-stretch",
+ # Disable by default for now because it requires some setups on host.
+ # "debian-stretch-arm64",
"ubuntu-xenial",
"ubuntu-bionic",
"ubuntu-cosmic",
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml
similarity index 74%
copy from dev/tasks/linux-packages/travis.linux.yml
copy to dev/tasks/linux-packages/travis.linux.arm64.yml
index c59ce08..853b706 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.arm64.yml
@@ -16,16 +16,23 @@
# under the License.
os: linux
-dist: trusty
-sudo: required
+dist: xenial
language: ruby
# don't build twice
if: tag IS blank
+services:
+ - docker
+
+addons:
+ apt:
+ packages:
+ - qemu-user-static
+ - unar
+
env:
global:
- - PLAT=x86_64
- TRAVIS_TAG={{ task.tag }}
- BUILD_REF={{ arrow.head }}
- ARROW_VERSION={{ arrow.version }}
@@ -39,6 +46,16 @@ script:
- pushd arrow/dev/tasks/linux-packages
- rake version:update
- rake dist
+ # We can remove this with binfmt-support 2.1.7 or later and
+ # qemu-user-static 2.12 or later. It requires Debian buster or later,
+ # or Ubuntu 18.10 or later.
+ - wget http://archive.ubuntu.com/ubuntu/pool/universe/q/qemu/qemu-user-static_3.1+dfsg-2ubuntu5_amd64.deb
+ - unar *.deb
+ - rm *.deb
+ - pushd qemu-user-static*
+ - unar data.*
+ - cp data/usr/bin/qemu-aarch64-static ../apt/debian-stretch-arm64/
+ - popd
- {{ build_command }}
deploy:
diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml
index c59ce08..937d4c7 100644
--- a/dev/tasks/linux-packages/travis.linux.yml
+++ b/dev/tasks/linux-packages/travis.linux.yml
@@ -16,16 +16,17 @@
# under the License.
os: linux
-dist: trusty
-sudo: required
+dist: xenial
language: ruby
+services:
+ - docker
+
# don't build twice
if: tag IS blank
env:
global:
- - PLAT=x86_64
- TRAVIS_TAG={{ task.tag }}
- BUILD_REF={{ arrow.head }}
- ARROW_VERSION={{ arrow.version }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 3e3acff..910650b 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -42,6 +42,8 @@ groups:
- wheel-win-cp37m
linux:
- debian-stretch
+ # We can't finish this on Travis CI for now.
+ # - debian-stretch-arm64
- ubuntu-xenial
- ubuntu-bionic
- ubuntu-cosmic
@@ -319,6 +321,44 @@ tasks:
- plasma-store-server-dbgsym_{no_rc_version}-1_amd64.deb
- plasma-store-server_{no_rc_version}-1_amd64.deb
+ # Don't use this for now. We can't complete this task in 50min on Travis CI.
+ debian-stretch-arm64:
+ platform: linux
+ template: linux-packages/travis.linux.arm64.yml
+ params:
+ build_command: rake apt:build APT_TARGETS=debian-stretch-arm64
+ upload_extensions:
+ - .deb
+ - .dsc
+ - .debian.tar.xz
+ - .orig.tar.gz
+ artifacts:
+ - apache-arrow_{no_rc_version}-1.debian.tar.xz
+ - apache-arrow_{no_rc_version}-1.dsc
+ - apache-arrow_{no_rc_version}.orig.tar.gz
+ - gir1.2-arrow-1.0_{no_rc_version}-1_amd64.deb
+ - gir1.2-parquet-1.0_{no_rc_version}-1_amd64.deb
+ - libarrow-dev_{no_rc_version}-1_amd64.deb
+ - libarrow-flight-dev_{no_rc_version}-1_amd64.deb
+ - libarrow-flight14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libarrow-flight14_{no_rc_version}-1_amd64.deb
+ - libarrow-glib-dev_{no_rc_version}-1_amd64.deb
+ - libarrow-glib-doc_{no_rc_version}-1_all.deb
+ - libarrow-glib14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libarrow-glib14_{no_rc_version}-1_amd64.deb
+ - libarrow-python-dev_{no_rc_version}-1_amd64.deb
+ - libarrow-python14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libarrow-python14_{no_rc_version}-1_amd64.deb
+ - libarrow14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libarrow14_{no_rc_version}-1_amd64.deb
+ - libparquet-dev_{no_rc_version}-1_amd64.deb
+ - libparquet-glib-dev_{no_rc_version}-1_amd64.deb
+ - libparquet-glib-doc_{no_rc_version}-1_all.deb
+ - libparquet-glib14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libparquet-glib14_{no_rc_version}-1_amd64.deb
+ - libparquet14-dbgsym_{no_rc_version}-1_amd64.deb
+ - libparquet14_{no_rc_version}-1_amd64.deb
+
ubuntu-xenial:
platform: linux
template: linux-packages/travis.linux.yml
@@ -371,7 +411,7 @@ tasks:
platform: linux
template: linux-packages/travis.linux.yml
params:
- build_command: travis_wait 45 rake apt:build APT_TARGETS=ubuntu-bionic
+ build_command: rake apt:build APT_TARGETS=ubuntu-bionic
upload_extensions:
- .deb
- .dsc
@@ -422,7 +462,7 @@ tasks:
platform: linux
template: linux-packages/travis.linux.yml
params:
- build_command: travis_wait 45 rake apt:build APT_TARGETS=ubuntu-cosmic
+ build_command: rake apt:build APT_TARGETS=ubuntu-cosmic
upload_extensions:
- .deb
- .dsc