You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2022/07/29 16:17:20 UTC
[arrow] branch master updated: ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 036fdf2d03 ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
036fdf2d03 is described below
commit 036fdf2d03c3a6986109f053d94fc237d2c2f82d
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sat Jul 30 01:17:14 2022 +0900
ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
Because system jemalloc can't be used with dlopen(). If system
jemalloc can't used with dlopen(), our shared libraried can't be
loaded as bindings of script languages such as Ruby:
+ ruby -r gi -e 'p GI.load('\''Arrow'\'')'
(null)-WARNING **: Failed to load shared library 'libarrow-glib.so.900' referenced by the typelib:
/lib64/libjemalloc.so.2: cannot allocate memory in static TLS block
This is caused because system jemalloc isn't built with
--disable-initial-exec-tls.
See also:
* https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=951704
* https://github.com/jemalloc/jemalloc/issues/1237
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Krisztián Szűcs <sz...@gmail.com>
---
dev/release/verify-apt.sh | 2 ++
dev/release/verify-yum.sh | 39 +++++++++++++++++++---
.../apache-arrow/apt/debian-bookworm/Dockerfile | 1 -
.../apache-arrow/apt/debian-bullseye/Dockerfile | 1 -
.../apache-arrow/apt/ubuntu-bionic/Dockerfile | 1 -
.../apache-arrow/apt/ubuntu-focal/Dockerfile | 1 -
.../apache-arrow/apt/ubuntu-jammy/Dockerfile | 1 -
.../linux-packages/apache-arrow/debian/control.in | 2 --
.../apache-arrow/debian/libarrow-dev.install | 1 -
dev/tasks/linux-packages/apache-arrow/debian/rules | 3 +-
.../apache-arrow/yum/almalinux-8/Dockerfile | 1 -
.../apache-arrow/yum/almalinux-9/Dockerfile | 1 -
.../apache-arrow/yum/amazon-linux-2/Dockerfile | 1 -
.../linux-packages/apache-arrow/yum/arrow.spec.in | 5 ---
.../apache-arrow/yum/centos-7/Dockerfile | 1 -
.../apache-arrow/yum/centos-8-stream/Dockerfile | 1 -
.../apache-arrow/yum/centos-9-stream/Dockerfile | 1 -
17 files changed, 37 insertions(+), 26 deletions(-)
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
index 846d9c7ec3..121abdbc6e 100755
--- a/dev/release/verify-apt.sh
+++ b/dev/release/verify-apt.sh
@@ -158,6 +158,8 @@ echo "::endgroup::"
echo "::group::Test Apache Arrow GLib"
+export G_DEBUG=fatal-warnings
+
${APT_INSTALL} libarrow-glib-dev=${package_version}
${APT_INSTALL} libarrow-glib-doc=${package_version}
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index 889dddd9d1..93bbcfe136 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -46,12 +46,14 @@ repository_version="${distribution_version}"
cmake_package=cmake
cmake_command=cmake
+have_arrow_libs=no
have_flight=yes
have_gandiva=yes
have_glib=yes
have_parquet=yes
have_python=yes
-have_arrow_libs=no
+have_ruby=yes
+ruby_devel_packages=(ruby-devel)
install_command="dnf install -y --enablerepo=crb"
uninstall_command="dnf remove -y"
clean_command="dnf clean"
@@ -63,6 +65,7 @@ case "${distribution}-${distribution_version}" in
almalinux-8)
distribution_prefix="almalinux"
have_arrow_libs=yes
+ ruby_devel_packages+=(redhat-rpm-config)
install_command="dnf install -y --enablerepo=powertools"
info_command="dnf info --enablerepo=powertools"
;;
@@ -73,12 +76,13 @@ case "${distribution}-${distribution_version}" in
distribution_prefix="amazon-linux"
cmake_package=cmake3
cmake_command=cmake3
- have_flight=no
- have_gandiva=no
- have_python=no
if [ "$(arch)" != "aarch64" ]; then
have_arrow_libs=yes
fi
+ have_flight=no
+ have_gandiva=no
+ have_python=no
+ have_ruby=no
install_command="yum install -y"
uninstall_command="yum remove -y"
clean_command="yum clean"
@@ -89,10 +93,11 @@ case "${distribution}-${distribution_version}" in
distribution_prefix="centos"
cmake_package=cmake3
cmake_command=cmake3
+ have_arrow_libs=yes
have_flight=no
have_gandiva=no
have_python=no
- have_arrow_libs=yes
+ have_ruby=no
install_command="yum install -y"
uninstall_command="yum remove -y"
clean_command="yum clean"
@@ -101,6 +106,7 @@ case "${distribution}-${distribution_version}" in
centos-8)
distribution_prefix="centos"
repository_version+="-stream"
+ ruby_devel_packages+=(redhat-rpm-config)
install_command="dnf install -y --enablerepo=powertools"
info_command="dnf info --enablerepo=powertools"
;;
@@ -206,6 +212,8 @@ echo "::endgroup::"
if [ "${have_glib}" = "yes" ]; then
echo "::group::Test Apache Arrow GLib"
+ export G_DEBUG=fatal-warnings
+
${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
@@ -215,6 +223,12 @@ if [ "${have_glib}" = "yes" ]; then
valac --pkg arrow-glib --pkg posix build.vala
./build
popd
+
+ if [ "${have_ruby}" = "yes" ]; then
+ ${install_command} "${ruby_devel_packages[@]}"
+ gem install gobject-introspection
+ ruby -r gi -e "p GI.load('Arrow')"
+ fi
echo "::endgroup::"
fi
@@ -222,11 +236,17 @@ if [ "${have_flight}" = "yes" ]; then
echo "::group::Test Apache Arrow Flight"
${install_command} --enablerepo=epel arrow-flight-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-flight-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('ArrowFlight')"
+ fi
echo "::endgroup::"
echo "::group::Test Apache Arrow Flight SQL"
${install_command} --enablerepo=epel arrow-flight-sql-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-flight-sql-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('ArrowFlightSQL')"
+ fi
echo "::endgroup::"
fi
@@ -240,6 +260,9 @@ echo "::group::Test Plasma"
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel plasma-glib-devel-${package_version}
${install_command} --enablerepo=epel plasma-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Plasma')"
+ fi
else
${install_command} --enablerepo=epel plasma-devel-${package_version}
fi
@@ -250,6 +273,9 @@ if [ "${have_gandiva}" = "yes" ]; then
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel gandiva-glib-devel-${package_version}
${install_command} --enablerepo=epel gandiva-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Gandiva')"
+ fi
else
${install_command} --enablerepo=epel gandiva-devel-${package_version}
fi
@@ -261,6 +287,9 @@ if [ "${have_parquet}" = "yes" ]; then
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel parquet-glib-devel-${package_version}
${install_command} --enablerepo=epel parquet-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Parquet')"
+ fi
else
${install_command} --enablerepo=epel parquet-devel-${package_version}
fi
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
index 42a7327614..152ac08a7c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
@@ -53,7 +53,6 @@ RUN \
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
index 1562f1d662..440a07f09c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
@@ -53,7 +53,6 @@ RUN \
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
index 64ebb3b72d..b897fa73a3 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -48,7 +48,6 @@ RUN \
libglib2.0-doc \
libgoogle-glog-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index 9efc560268..5abee7c2e0 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -50,7 +50,6 @@ RUN \
libgmock-dev \
libgoogle-glog-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
index e6f657533f..7e95c37642 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -51,7 +51,6 @@ RUN \
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
index 8de71bda75..e0a6378828 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control.in
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -18,7 +18,6 @@ Build-Depends:
libgoogle-glog-dev,
@USE_SYSTEM_GRPC@ libgrpc++-dev,
libgtest-dev,
- libjemalloc-dev,
liblz4-dev,
@USE_SYSTEM_GRPC@ libprotoc-dev,
@USE_SYSTEM_GRPC@ libprotobuf-dev,
@@ -147,7 +146,6 @@ Depends:
libbz2-dev,
@USE_SYSTEM_C_ARES@ libc-ares-dev,
@USE_SYSTEM_GRPC@ libgrpc++-dev,
- libjemalloc-dev,
liblz4-dev,
libre2-dev,
libsnappy-dev,
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
index b55c76d692..40a834ced4 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
@@ -4,7 +4,6 @@ usr/lib/*/cmake/arrow/ArrowOptions.cmake
usr/lib/*/cmake/arrow/ArrowTargets*.cmake
usr/lib/*/cmake/arrow/Find*Alt.cmake
usr/lib/*/cmake/arrow/FindArrow.cmake
-usr/lib/*/cmake/arrow/Findjemalloc.cmake
usr/lib/*/cmake/arrow/Find[BTuz]*.cmake
usr/lib/*/cmake/arrow/arrow-config.cmake
usr/lib/*/libarrow.a
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
index 9178111eba..2f3b694d68 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -62,8 +62,7 @@ override_dh_auto_configure:
-DCUDAToolkit_ROOT=/usr \
-DPARQUET_REQUIRE_ENCRYPTION=ON \
-DPythonInterp_FIND_VERSION=ON \
- -DPythonInterp_FIND_VERSION_MAJOR=3 \
- -Djemalloc_SOURCE=SYSTEM
+ -DPythonInterp_FIND_VERSION_MAJOR=3
override_dh_auto_build:
dh_auto_build \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
index 0a37bab4b1..a75fd022bd 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -40,7 +40,6 @@ RUN \
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
index 8561779f65..df63f7a1a7 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
@@ -39,7 +39,6 @@ RUN \
git \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
index fa1ded66bc..68d604d64e 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
@@ -37,7 +37,6 @@ RUN \
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
lz4-devel \
make \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 78f974f971..098342bacd 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -105,7 +105,6 @@ BuildRequires: git
%if %{use_glog}
BuildRequires: glog-devel
%endif
-BuildRequires: jemalloc-devel
%if %{use_gcs}
BuildRequires: json-devel
%endif
@@ -194,7 +193,6 @@ cd cpp
-DPythonInterp_FIND_VERSION=ON \
-DPythonInterp_FIND_VERSION_MAJOR=3 \
%endif
- -Djemalloc_SOURCE=SYSTEM \
-G"Unix Makefiles"
%arrow_cmake_build
cd -
@@ -243,7 +241,6 @@ Requires: gflags
%if %{use_glog}
Requires: glog
%endif
-Requires: jemalloc
%if %{have_zstd}
Requires: libzstd
%endif
@@ -275,7 +272,6 @@ Requires: bzip2-devel
%if %{use_flight}
Requires: c-ares-devel
%endif
-Requires: jemalloc-devel
%if %{use_gcs}
Requires: json-devel
%endif
@@ -323,7 +319,6 @@ Libraries and header files for Apache Arrow C++.
%if %{use_flight}
%{_libdir}/cmake/arrow/Findc-aresAlt.cmake
%endif
-%{_libdir}/cmake/arrow/Findjemalloc.cmake
%{_libdir}/cmake/arrow/Findlz4Alt.cmake
%if %{have_re2}
%{_libdir}/cmake/arrow/Findre2Alt.cmake
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
index 93d4a7812b..9c93e2f240 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
@@ -39,7 +39,6 @@ RUN \
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libzstd-devel \
lz4-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
index ca9e44d409..b29cc4565b 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
@@ -40,7 +40,6 @@ RUN \
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
index 2bc036432c..38e6ae4531 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
@@ -39,7 +39,6 @@ RUN \
git \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \