You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2018/09/29 16:51:04 UTC
[arrow] branch master updated: ARROW-3276: [Packaging] Add support
for Parquet deb/rpm packages
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 7157a74 ARROW-3276: [Packaging] Add support for Parquet deb/rpm packages
7157a74 is described below
commit 7157a7454e60a0d5b7eb7886e72444130ceb17a1
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Sat Sep 29 18:50:36 2018 +0200
ARROW-3276: [Packaging] Add support for Parquet deb/rpm packages
Author: Kouhei Sutou <ko...@clear-code.com>
Author: Krisztián Szűcs <sz...@gmail.com>
Closes #2603 from kou/packages-linux-parquet and squashes the following commits:
04b50179 <Krisztián Szűcs> fix typo
324c1200 <Kouhei Sutou> Add support for Parquet deb/rpm packages
---
dev/release/rat_exclude_files.txt | 4 ++
.../linux-packages/apt/debian-stretch/Dockerfile | 2 +
.../linux-packages/apt/ubuntu-bionic/Dockerfile | 2 +
.../linux-packages/apt/ubuntu-trusty/Dockerfile | 2 +
.../linux-packages/apt/ubuntu-xenial/Dockerfile | 2 +
.../linux-packages/debian.ubuntu-trusty/control | 38 ++++++++++++++---
.../debian.ubuntu-trusty/libparquet-dev.install | 4 ++
.../debian.ubuntu-trusty/libparquet11.install | 1 +
.../linux-packages/debian.ubuntu-trusty/rules | 3 +-
dev/tasks/linux-packages/debian/control | 44 ++++++++++++++++----
.../linux-packages/debian/libparquet-dev.install | 4 ++
.../linux-packages/debian/libparquet11.install | 1 +
dev/tasks/linux-packages/debian/rules | 1 +
dev/tasks/linux-packages/yum/arrow.spec.in | 48 +++++++++++++++++++++-
dev/tasks/linux-packages/yum/centos-7/Dockerfile | 2 +
dev/tasks/tasks.yml | 11 +++++
16 files changed, 151 insertions(+), 18 deletions(-)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index e366ae3..2948b01 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -54,6 +54,8 @@ dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-glib-doc.links
dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-glib11.install
dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-python11.install
dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow11.install
+dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
+dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
dev/tasks/linux-packages/debian.ubuntu-trusty/patches/series
dev/tasks/linux-packages/debian.ubuntu-trusty/rules
dev/tasks/linux-packages/debian.ubuntu-trusty/source/format
@@ -75,6 +77,8 @@ dev/tasks/linux-packages/debian/libarrow-gpu11.install
dev/tasks/linux-packages/debian/libarrow-python-dev.install
dev/tasks/linux-packages/debian/libarrow-python11.install
dev/tasks/linux-packages/debian/libarrow11.install
+dev/tasks/linux-packages/debian/libparquet-dev.install
+dev/tasks/linux-packages/debian/libparquet11.install
dev/tasks/linux-packages/debian/patches/series
dev/tasks/linux-packages/debian/rules
dev/tasks/linux-packages/debian/source/format
diff --git a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
index fc14761..24a7ddf 100644
--- a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
+++ b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
@@ -28,10 +28,12 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
autoconf-archive \
+ bison \
build-essential \
cmake \
debhelper\
devscripts \
+ flex \
git \
gtk-doc-tools \
libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
index 2452bac..dcf7836 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
@@ -26,10 +26,12 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
autoconf-archive \
+ bison \
build-essential \
cmake \
debhelper\
devscripts \
+ flex \
git \
gtk-doc-tools \
libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
index 1273287..727e035 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
@@ -26,11 +26,13 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
autoconf-archive \
+ bison \
build-essential \
cmake3 \
debhelper\
devscripts \
dh-autoreconf \
+ flex \
git \
gtk-doc-tools \
libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
index 2449f28..93a6d3a 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
@@ -26,11 +26,13 @@ RUN \
apt update ${quiet} && \
apt install -y -V ${quiet} \
autoconf-archive \
+ bison \
build-essential \
cmake \
debhelper\
devscripts \
dh-autoreconf \
+ flex \
git \
gtk-doc-tools \
libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/control b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
index 4551d0b..c4b16ac 100644
--- a/dev/tasks/linux-packages/debian.ubuntu-trusty/control
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
@@ -3,18 +3,20 @@ Section: devel
Priority: optional
Maintainer: Kouhei Sutou <ko...@clear-code.com>
Build-Depends:
+ bison,
+ cmake,
debhelper (>= 9),
dh-autoreconf,
- pkg-config,
- cmake,
+ flex,
git,
- libboost-system-dev,
+ gobject-introspection,
+ gtk-doc-tools,
libboost-filesystem-dev,
libboost-regex-dev,
- libjemalloc-dev,
+ libboost-system-dev,
libgirepository1.0-dev,
- gtk-doc-tools,
- gobject-introspection
+ libjemalloc-dev,
+ pkg-config
Build-Depends-Indep: libglib2.0-doc
Standards-Version: 3.9.6
Homepage: https://arrow.apache.org/
@@ -42,6 +44,30 @@ Description: Apache Arrow is a data processing library for analysis
.
This package provides header files.
+Package: libparquet11
+Section: libs
+Architecture: any
+Multi-Arch: same
+Pre-Depends: ${misc:Pre-Depends}
+Depends:
+ ${misc:Depends},
+ ${shlibs:Depends}
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides library files to process Apache Parquet format.
+
+Package: libparquet-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ libarrow-dev (= ${binary:Version}),
+ libparquet11 (= ${binary:Version})
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides header files.
+
Package: libarrow-glib11
Section: libs
Architecture: any
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
new file mode 100644
index 0000000..523642e
--- /dev/null
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
@@ -0,0 +1,4 @@
+usr/include/parquet/
+usr/lib/*/libparquet.a
+usr/lib/*/libparquet.so
+usr/lib/*/pkgconfig/parquet.pc
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
new file mode 100644
index 0000000..540a91d
--- /dev/null
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
@@ -0,0 +1 @@
+usr/lib/*/libparquet.so.*
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/rules b/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
index 5dc873f..01956fe 100755
--- a/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
@@ -23,7 +23,8 @@ override_dh_auto_configure:
-- \
-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
-DARROW_BUILD_TESTS=OFF \
- -DARROW_ORC=ON
+ -DARROW_ORC=ON \
+ -DARROW_PARQUET=ON
dh_auto_configure \
--sourcedirectory=c_glib \
--builddirectory=c_glib_build \
diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control
index 8daaa23..4391e1c 100644
--- a/dev/tasks/linux-packages/debian/control
+++ b/dev/tasks/linux-packages/debian/control
@@ -3,25 +3,27 @@ Section: devel
Priority: optional
Maintainer: Kouhei Sutou <ko...@clear-code.com>
Build-Depends:
- debhelper (>= 9.20160115),
- dh-autoreconf,
autoconf-archive,
- pkg-config,
+ bison,
cmake,
+ debhelper (>= 9.20160115),
+ dh-autoreconf,
+ flex,
git,
- libboost-system-dev,
+ gobject-introspection,
+ gtk-doc-tools,
libboost-filesystem-dev,
libboost-regex-dev,
+ libboost-system-dev,
+ libgirepository1.0-dev,
libjemalloc-dev,
libprotobuf-dev,
libprotoc-dev,
+ nvidia-cuda-toolkit,
+ pkg-config,
protobuf-compiler,
python3-dev,
- python3-numpy,
- nvidia-cuda-toolkit,
- libgirepository1.0-dev,
- gtk-doc-tools,
- gobject-introspection
+ python3-numpy
Build-Depends-Indep: libglib2.0-doc
Standards-Version: 3.9.6
Homepage: https://arrow.apache.org/
@@ -101,6 +103,30 @@ Description: Apache Arrow is a data processing library for analysis
.
This package provides header files for GPU support.
+Package: libparquet11
+Section: libs
+Architecture: any
+Multi-Arch: same
+Pre-Depends: ${misc:Pre-Depends}
+Depends:
+ ${misc:Depends},
+ ${shlibs:Depends}
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides library files to process Apache Parquet format.
+
+Package: libparquet-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ libarrow-dev (= ${binary:Version}),
+ libparquet11 (= ${binary:Version})
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides header files.
+
Package: libarrow-glib11
Section: libs
Architecture: any
diff --git a/dev/tasks/linux-packages/debian/libparquet-dev.install b/dev/tasks/linux-packages/debian/libparquet-dev.install
new file mode 100644
index 0000000..523642e
--- /dev/null
+++ b/dev/tasks/linux-packages/debian/libparquet-dev.install
@@ -0,0 +1,4 @@
+usr/include/parquet/
+usr/lib/*/libparquet.a
+usr/lib/*/libparquet.so
+usr/lib/*/pkgconfig/parquet.pc
diff --git a/dev/tasks/linux-packages/debian/libparquet11.install b/dev/tasks/linux-packages/debian/libparquet11.install
new file mode 100644
index 0000000..540a91d
--- /dev/null
+++ b/dev/tasks/linux-packages/debian/libparquet11.install
@@ -0,0 +1 @@
+usr/lib/*/libparquet.so.*
diff --git a/dev/tasks/linux-packages/debian/rules b/dev/tasks/linux-packages/debian/rules
index 3041d24..ce39fde 100755
--- a/dev/tasks/linux-packages/debian/rules
+++ b/dev/tasks/linux-packages/debian/rules
@@ -28,6 +28,7 @@ override_dh_auto_configure:
-DARROW_PYTHON=ON \
-DARROW_BOOST_USE_SHARED=ON \
-DARROW_ORC=ON \
+ -DARROW_PARQUET=ON \
-DPROTOBUF_HOME=/usr \
-DARROW_PROTOBUF_USE_SHARED=ON \
-DPythonInterp_FIND_VERSION=ON \
diff --git a/dev/tasks/linux-packages/yum/arrow.spec.in b/dev/tasks/linux-packages/yum/arrow.spec.in
index 1b9cd86..d42c39e 100644
--- a/dev/tasks/linux-packages/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/yum/arrow.spec.in
@@ -20,6 +20,7 @@
%define _centos_ver %{?centos_ver:%{centos_ver}}%{!?centos_ver:7}
%define use_python %{_centos_ver} >= 6
+%define use_parquet %{_centos_ver} >= 7
%define use_glib %{_centos_ver} >= 7
Name: @PACKAGE@
@@ -41,6 +42,10 @@ BuildRequires: jemalloc-devel
BuildRequires: python34-devel
BuildRequires: python34-numpy
%endif
+%if %{use_parquet}
+BuildRequires: bison
+BuildRequires: flex
+%endif
%if %{use_glib}
BuildRequires: autoconf-archive
BuildRequires: gtk-doc
@@ -68,6 +73,9 @@ cd cpp/build
-DARROW_BOOST_VENDORED=ON \
%endif
-DARROW_ORC=ON \
+%if %{use_parquet}
+ -DARROW_PARQUET=ON \
+%endif
-DARROW_BUILD_TESTS=OFF
make %{?_smp_mflags}
cd -
@@ -136,7 +144,7 @@ Libraries and header files for Apache Arrow C++.
%package python-libs
Summary: Apache Arrow CPython extensions
License: Apache-2.0
-Requires: arrow-libs = %{version}-%{release}
+Requires: %{name}-libs = %{version}-%{release}
Requires: python34-numpy
%description python-libs
@@ -150,7 +158,7 @@ This package contains the Apache Arrow CPython extensions.
%package python-devel
Summary: Libraries and header files for Apache Arrow CPython extensions
License: Apache-2.0
-Requires: arrow-devel = %{version}-%{release}
+Requires: %{name}-devel = %{version}-%{release}
Requires: %{name}-libs = %{version}-%{release}
Requires: python34-devel
@@ -165,6 +173,42 @@ Libraries and header files for Apache Arrow CPython extensions.
%{_libdir}/pkgconfig/arrow-python.pc
%endif
+%if %{use_parquet}
+%package -n parquet-libs
+Summary: Runtime libraries for Apache Parquet
+License: Apache-2.0
+Requires: boost-system
+Requires: boost-filesystem
+Requires: boost-program-options
+Requires: boost-regex
+Requires: %{name}-libs = %{version}-%{release}
+
+%description -n parquet-libs
+This package contains the libraries for Apache Parquet
+
+%files -n parquet-libs
+%defattr(-,root,root,-)
+%doc README.md LICENSE.txt NOTICE.txt
+%{_libdir}/libparquet.so.*
+
+%package -n parquet-devel
+Summary: Libraries and header files for Apache Parquet
+License: Apache-2.0
+Requires: parquet-libs = %{version}-%{release}
+Requires: zlib-devel
+
+%description -n parquet-devel
+Libraries and header files for Apache Parquet.
+
+%files -n parquet-devel
+%defattr(-,root,root,-)
+%doc README.md LICENSE.txt NOTICE.txt
+%{_includedir}/parquet/
+%{_libdir}/libparquet.a
+%{_libdir}/libparquet.so
+%{_libdir}/pkgconfig/parquet*.pc
+%endif
+
%if %{use_glib}
%package glib-libs
Summary: Runtime libraries for Apache Arrow GLib
diff --git a/dev/tasks/linux-packages/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
index 1311a45..a8710c5 100644
--- a/dev/tasks/linux-packages/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
@@ -26,8 +26,10 @@ RUN \
yum groupinstall -y ${quiet} "Development Tools" && \
yum install -y ${quiet} \
autoconf-archive \
+ bison \
boost-devel \
cmake3 \
+ flex \
git \
gobject-introspection-devel \
gtk-doc \
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 30403e1..991e900 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -256,6 +256,9 @@ tasks:
- libarrow-python11_{version}-1_amd64.deb
- libarrow11-dbgsym_{version}-1_amd64.deb
- libarrow11_{version}-1_amd64.deb
+ - libparquet-dev_{version}-1_amd64.deb
+ - libparquet11-dbgsym_{version}-1_amd64.deb
+ - libparquet11_{version}-1_amd64.deb
ubuntu-trusty:
platform: linux
@@ -277,6 +280,8 @@ tasks:
- libarrow-glib-doc_{version}-1_all.deb
- libarrow-glib11_{version}-1_amd64.deb
- libarrow11_{version}-1_amd64.deb
+ - libparquet-dev_{version}-1_amd64.deb
+ - libparquet11_{version}-1_amd64.deb
ubuntu-xenial:
platform: linux
@@ -305,6 +310,8 @@ tasks:
- libarrow-python-dev_{version}-1_amd64.deb
- libarrow-python11_{version}-1_amd64.deb
- libarrow11_{version}-1_amd64.deb
+ - libparquet-dev_{version}-1_amd64.deb
+ - libparquet11_{version}-1_amd64.deb
ubuntu-bionic:
platform: linux
@@ -333,6 +340,8 @@ tasks:
- libarrow-python-dev_{version}-1_amd64.deb
- libarrow-python11_{version}-1_amd64.deb
- libarrow11_{version}-1_amd64.deb
+ - libparquet-dev_{version}-1_amd64.deb
+ - libparquet11_{version}-1_amd64.deb
centos-6:
platform: linux
@@ -366,3 +375,5 @@ tasks:
- arrow-libs-{version}-1.el7.x86_64.rpm
- arrow-python-devel-{version}-1.el7.x86_64.rpm
- arrow-python-libs-{version}-1.el7.x86_64.rpm
+ - parquet-devel-{version}-1.el7.x86_64.rpm
+ - parquet-libs-{version}-1.el7.x86_64.rpm