You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2018/09/29 16:51:04 UTC

[arrow] branch master updated: ARROW-3276: [Packaging] Add support for Parquet deb/rpm packages

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 7157a74  ARROW-3276: [Packaging] Add support for Parquet deb/rpm packages
7157a74 is described below

commit 7157a7454e60a0d5b7eb7886e72444130ceb17a1
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Sat Sep 29 18:50:36 2018 +0200

    ARROW-3276: [Packaging] Add support for Parquet deb/rpm packages
    
    Author: Kouhei Sutou <ko...@clear-code.com>
    Author: Krisztián Szűcs <sz...@gmail.com>
    
    Closes #2603 from kou/packages-linux-parquet and squashes the following commits:
    
    04b50179 <Krisztián Szűcs>  fix typo
    324c1200 <Kouhei Sutou>  Add support for Parquet deb/rpm packages
---
 dev/release/rat_exclude_files.txt                  |  4 ++
 .../linux-packages/apt/debian-stretch/Dockerfile   |  2 +
 .../linux-packages/apt/ubuntu-bionic/Dockerfile    |  2 +
 .../linux-packages/apt/ubuntu-trusty/Dockerfile    |  2 +
 .../linux-packages/apt/ubuntu-xenial/Dockerfile    |  2 +
 .../linux-packages/debian.ubuntu-trusty/control    | 38 ++++++++++++++---
 .../debian.ubuntu-trusty/libparquet-dev.install    |  4 ++
 .../debian.ubuntu-trusty/libparquet11.install      |  1 +
 .../linux-packages/debian.ubuntu-trusty/rules      |  3 +-
 dev/tasks/linux-packages/debian/control            | 44 ++++++++++++++++----
 .../linux-packages/debian/libparquet-dev.install   |  4 ++
 .../linux-packages/debian/libparquet11.install     |  1 +
 dev/tasks/linux-packages/debian/rules              |  1 +
 dev/tasks/linux-packages/yum/arrow.spec.in         | 48 +++++++++++++++++++++-
 dev/tasks/linux-packages/yum/centos-7/Dockerfile   |  2 +
 dev/tasks/tasks.yml                                | 11 +++++
 16 files changed, 151 insertions(+), 18 deletions(-)

diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index e366ae3..2948b01 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -54,6 +54,8 @@ dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-glib-doc.links
 dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-glib11.install
 dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow-python11.install
 dev/tasks/linux-packages/debian.ubuntu-trusty/libarrow11.install
+dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
+dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
 dev/tasks/linux-packages/debian.ubuntu-trusty/patches/series
 dev/tasks/linux-packages/debian.ubuntu-trusty/rules
 dev/tasks/linux-packages/debian.ubuntu-trusty/source/format
@@ -75,6 +77,8 @@ dev/tasks/linux-packages/debian/libarrow-gpu11.install
 dev/tasks/linux-packages/debian/libarrow-python-dev.install
 dev/tasks/linux-packages/debian/libarrow-python11.install
 dev/tasks/linux-packages/debian/libarrow11.install
+dev/tasks/linux-packages/debian/libparquet-dev.install
+dev/tasks/linux-packages/debian/libparquet11.install
 dev/tasks/linux-packages/debian/patches/series
 dev/tasks/linux-packages/debian/rules
 dev/tasks/linux-packages/debian/source/format
diff --git a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
index fc14761..24a7ddf 100644
--- a/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
+++ b/dev/tasks/linux-packages/apt/debian-stretch/Dockerfile
@@ -28,10 +28,12 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     autoconf-archive \
+    bison \
     build-essential \
     cmake \
     debhelper\
     devscripts \
+    flex \
     git \
     gtk-doc-tools \
     libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
index 2452bac..dcf7836 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-bionic/Dockerfile
@@ -26,10 +26,12 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     autoconf-archive \
+    bison \
     build-essential \
     cmake \
     debhelper\
     devscripts \
+    flex \
     git \
     gtk-doc-tools \
     libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
index 1273287..727e035 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-trusty/Dockerfile
@@ -26,11 +26,13 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     autoconf-archive \
+    bison \
     build-essential \
     cmake3 \
     debhelper\
     devscripts \
     dh-autoreconf \
+    flex \
     git \
     gtk-doc-tools \
     libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
index 2449f28..93a6d3a 100644
--- a/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
+++ b/dev/tasks/linux-packages/apt/ubuntu-xenial/Dockerfile
@@ -26,11 +26,13 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     autoconf-archive \
+    bison \
     build-essential \
     cmake \
     debhelper\
     devscripts \
     dh-autoreconf \
+    flex \
     git \
     gtk-doc-tools \
     libboost-filesystem-dev \
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/control b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
index 4551d0b..c4b16ac 100644
--- a/dev/tasks/linux-packages/debian.ubuntu-trusty/control
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/control
@@ -3,18 +3,20 @@ Section: devel
 Priority: optional
 Maintainer: Kouhei Sutou <ko...@clear-code.com>
 Build-Depends:
+  bison,
+  cmake,
   debhelper (>= 9),
   dh-autoreconf,
-  pkg-config,
-  cmake,
+  flex,
   git,
-  libboost-system-dev,
+  gobject-introspection,
+  gtk-doc-tools,
   libboost-filesystem-dev,
   libboost-regex-dev,
-  libjemalloc-dev,
+  libboost-system-dev,
   libgirepository1.0-dev,
-  gtk-doc-tools,
-  gobject-introspection
+  libjemalloc-dev,
+  pkg-config
 Build-Depends-Indep: libglib2.0-doc
 Standards-Version: 3.9.6
 Homepage: https://arrow.apache.org/
@@ -42,6 +44,30 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides header files.
 
+Package: libparquet11
+Section: libs
+Architecture: any
+Multi-Arch: same
+Pre-Depends: ${misc:Pre-Depends}
+Depends:
+  ${misc:Depends},
+  ${shlibs:Depends}
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides library files to process Apache Parquet format.
+
+Package: libparquet-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends:
+  ${misc:Depends},
+  libarrow-dev (= ${binary:Version}),
+  libparquet11 (= ${binary:Version})
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides header files.
+
 Package: libarrow-glib11
 Section: libs
 Architecture: any
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
new file mode 100644
index 0000000..523642e
--- /dev/null
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet-dev.install
@@ -0,0 +1,4 @@
+usr/include/parquet/
+usr/lib/*/libparquet.a
+usr/lib/*/libparquet.so
+usr/lib/*/pkgconfig/parquet.pc
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
new file mode 100644
index 0000000..540a91d
--- /dev/null
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/libparquet11.install
@@ -0,0 +1 @@
+usr/lib/*/libparquet.so.*
diff --git a/dev/tasks/linux-packages/debian.ubuntu-trusty/rules b/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
index 5dc873f..01956fe 100755
--- a/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
+++ b/dev/tasks/linux-packages/debian.ubuntu-trusty/rules
@@ -23,7 +23,8 @@ override_dh_auto_configure:
 	  -- \
 	  -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
 	  -DARROW_BUILD_TESTS=OFF \
-	  -DARROW_ORC=ON
+	  -DARROW_ORC=ON \
+	  -DARROW_PARQUET=ON
 	dh_auto_configure \
 	  --sourcedirectory=c_glib \
 	  --builddirectory=c_glib_build \
diff --git a/dev/tasks/linux-packages/debian/control b/dev/tasks/linux-packages/debian/control
index 8daaa23..4391e1c 100644
--- a/dev/tasks/linux-packages/debian/control
+++ b/dev/tasks/linux-packages/debian/control
@@ -3,25 +3,27 @@ Section: devel
 Priority: optional
 Maintainer: Kouhei Sutou <ko...@clear-code.com>
 Build-Depends:
-  debhelper (>= 9.20160115),
-  dh-autoreconf,
   autoconf-archive,
-  pkg-config,
+  bison,
   cmake,
+  debhelper (>= 9.20160115),
+  dh-autoreconf,
+  flex,
   git,
-  libboost-system-dev,
+  gobject-introspection,
+  gtk-doc-tools,
   libboost-filesystem-dev,
   libboost-regex-dev,
+  libboost-system-dev,
+  libgirepository1.0-dev,
   libjemalloc-dev,
   libprotobuf-dev,
   libprotoc-dev,
+  nvidia-cuda-toolkit,
+  pkg-config,
   protobuf-compiler,
   python3-dev,
-  python3-numpy,
-  nvidia-cuda-toolkit,
-  libgirepository1.0-dev,
-  gtk-doc-tools,
-  gobject-introspection
+  python3-numpy
 Build-Depends-Indep: libglib2.0-doc
 Standards-Version: 3.9.6
 Homepage: https://arrow.apache.org/
@@ -101,6 +103,30 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides header files for GPU support.
 
+Package: libparquet11
+Section: libs
+Architecture: any
+Multi-Arch: same
+Pre-Depends: ${misc:Pre-Depends}
+Depends:
+  ${misc:Depends},
+  ${shlibs:Depends}
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides library files to process Apache Parquet format.
+
+Package: libparquet-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends:
+  ${misc:Depends},
+  libarrow-dev (= ${binary:Version}),
+  libparquet11 (= ${binary:Version})
+Description: Apache Parquet is a columnar storage format
+ .
+ This package provides header files.
+
 Package: libarrow-glib11
 Section: libs
 Architecture: any
diff --git a/dev/tasks/linux-packages/debian/libparquet-dev.install b/dev/tasks/linux-packages/debian/libparquet-dev.install
new file mode 100644
index 0000000..523642e
--- /dev/null
+++ b/dev/tasks/linux-packages/debian/libparquet-dev.install
@@ -0,0 +1,4 @@
+usr/include/parquet/
+usr/lib/*/libparquet.a
+usr/lib/*/libparquet.so
+usr/lib/*/pkgconfig/parquet.pc
diff --git a/dev/tasks/linux-packages/debian/libparquet11.install b/dev/tasks/linux-packages/debian/libparquet11.install
new file mode 100644
index 0000000..540a91d
--- /dev/null
+++ b/dev/tasks/linux-packages/debian/libparquet11.install
@@ -0,0 +1 @@
+usr/lib/*/libparquet.so.*
diff --git a/dev/tasks/linux-packages/debian/rules b/dev/tasks/linux-packages/debian/rules
index 3041d24..ce39fde 100755
--- a/dev/tasks/linux-packages/debian/rules
+++ b/dev/tasks/linux-packages/debian/rules
@@ -28,6 +28,7 @@ override_dh_auto_configure:
 	  -DARROW_PYTHON=ON \
 	  -DARROW_BOOST_USE_SHARED=ON \
 	  -DARROW_ORC=ON \
+	  -DARROW_PARQUET=ON \
 	  -DPROTOBUF_HOME=/usr \
 	  -DARROW_PROTOBUF_USE_SHARED=ON \
 	  -DPythonInterp_FIND_VERSION=ON \
diff --git a/dev/tasks/linux-packages/yum/arrow.spec.in b/dev/tasks/linux-packages/yum/arrow.spec.in
index 1b9cd86..d42c39e 100644
--- a/dev/tasks/linux-packages/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/yum/arrow.spec.in
@@ -20,6 +20,7 @@
 %define _centos_ver %{?centos_ver:%{centos_ver}}%{!?centos_ver:7}
 
 %define use_python %{_centos_ver} >= 6
+%define use_parquet %{_centos_ver} >= 7
 %define use_glib %{_centos_ver} >= 7
 
 Name:		@PACKAGE@
@@ -41,6 +42,10 @@ BuildRequires:	jemalloc-devel
 BuildRequires:	python34-devel
 BuildRequires:	python34-numpy
 %endif
+%if %{use_parquet}
+BuildRequires:	bison
+BuildRequires:	flex
+%endif
 %if %{use_glib}
 BuildRequires:	autoconf-archive
 BuildRequires:	gtk-doc
@@ -68,6 +73,9 @@ cd cpp/build
   -DARROW_BOOST_VENDORED=ON \
 %endif
   -DARROW_ORC=ON \
+%if %{use_parquet}
+  -DARROW_PARQUET=ON \
+%endif
   -DARROW_BUILD_TESTS=OFF
 make %{?_smp_mflags}
 cd -
@@ -136,7 +144,7 @@ Libraries and header files for Apache Arrow C++.
 %package python-libs
 Summary:	Apache Arrow CPython extensions
 License:	Apache-2.0
-Requires:	arrow-libs = %{version}-%{release}
+Requires:	%{name}-libs = %{version}-%{release}
 Requires:	python34-numpy
 
 %description python-libs
@@ -150,7 +158,7 @@ This package contains the Apache Arrow CPython extensions.
 %package python-devel
 Summary:	Libraries and header files for Apache Arrow CPython extensions
 License:	Apache-2.0
-Requires:	arrow-devel = %{version}-%{release}
+Requires:	%{name}-devel = %{version}-%{release}
 Requires:	%{name}-libs = %{version}-%{release}
 Requires:	python34-devel
 
@@ -165,6 +173,42 @@ Libraries and header files for Apache Arrow CPython extensions.
 %{_libdir}/pkgconfig/arrow-python.pc
 %endif
 
+%if %{use_parquet}
+%package -n parquet-libs
+Summary:	Runtime libraries for Apache Parquet
+License:	Apache-2.0
+Requires:	boost-system
+Requires:	boost-filesystem
+Requires:	boost-program-options
+Requires:	boost-regex
+Requires:	%{name}-libs = %{version}-%{release}
+
+%description -n parquet-libs
+This package contains the libraries for Apache Parquet
+
+%files -n parquet-libs
+%defattr(-,root,root,-)
+%doc README.md LICENSE.txt NOTICE.txt
+%{_libdir}/libparquet.so.*
+
+%package -n parquet-devel
+Summary:	Libraries and header files for Apache Parquet
+License:	Apache-2.0
+Requires:	parquet-libs = %{version}-%{release}
+Requires:	zlib-devel
+
+%description -n parquet-devel
+Libraries and header files for Apache Parquet.
+
+%files -n parquet-devel
+%defattr(-,root,root,-)
+%doc README.md LICENSE.txt NOTICE.txt
+%{_includedir}/parquet/
+%{_libdir}/libparquet.a
+%{_libdir}/libparquet.so
+%{_libdir}/pkgconfig/parquet*.pc
+%endif
+
 %if %{use_glib}
 %package glib-libs
 Summary:	Runtime libraries for Apache Arrow GLib
diff --git a/dev/tasks/linux-packages/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
index 1311a45..a8710c5 100644
--- a/dev/tasks/linux-packages/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/yum/centos-7/Dockerfile
@@ -26,8 +26,10 @@ RUN \
   yum groupinstall -y ${quiet} "Development Tools" && \
   yum install -y ${quiet} \
     autoconf-archive \
+    bison \
     boost-devel \
     cmake3 \
+    flex \
     git \
     gobject-introspection-devel \
     gtk-doc \
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 30403e1..991e900 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -256,6 +256,9 @@ tasks:
       - libarrow-python11_{version}-1_amd64.deb
       - libarrow11-dbgsym_{version}-1_amd64.deb
       - libarrow11_{version}-1_amd64.deb
+      - libparquet-dev_{version}-1_amd64.deb
+      - libparquet11-dbgsym_{version}-1_amd64.deb
+      - libparquet11_{version}-1_amd64.deb
 
   ubuntu-trusty:
     platform: linux
@@ -277,6 +280,8 @@ tasks:
       - libarrow-glib-doc_{version}-1_all.deb
       - libarrow-glib11_{version}-1_amd64.deb
       - libarrow11_{version}-1_amd64.deb
+      - libparquet-dev_{version}-1_amd64.deb
+      - libparquet11_{version}-1_amd64.deb
 
   ubuntu-xenial:
     platform: linux
@@ -305,6 +310,8 @@ tasks:
       - libarrow-python-dev_{version}-1_amd64.deb
       - libarrow-python11_{version}-1_amd64.deb
       - libarrow11_{version}-1_amd64.deb
+      - libparquet-dev_{version}-1_amd64.deb
+      - libparquet11_{version}-1_amd64.deb
 
   ubuntu-bionic:
     platform: linux
@@ -333,6 +340,8 @@ tasks:
       - libarrow-python-dev_{version}-1_amd64.deb
       - libarrow-python11_{version}-1_amd64.deb
       - libarrow11_{version}-1_amd64.deb
+      - libparquet-dev_{version}-1_amd64.deb
+      - libparquet11_{version}-1_amd64.deb
 
   centos-6:
     platform: linux
@@ -366,3 +375,5 @@ tasks:
       - arrow-libs-{version}-1.el7.x86_64.rpm
       - arrow-python-devel-{version}-1.el7.x86_64.rpm
       - arrow-python-libs-{version}-1.el7.x86_64.rpm
+      - parquet-devel-{version}-1.el7.x86_64.rpm
+      - parquet-libs-{version}-1.el7.x86_64.rpm