You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2022/07/29 16:17:20 UTC

[arrow] branch master updated: ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 036fdf2d03 ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
036fdf2d03 is described below

commit 036fdf2d03c3a6986109f053d94fc237d2c2f82d
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sat Jul 30 01:17:14 2022 +0900

    ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
    
    Because system jemalloc can't be used with dlopen(). If system
    jemalloc can't used with dlopen(), our shared libraried can't be
    loaded as bindings of script languages such as Ruby:
    
        + ruby -r gi -e 'p GI.load('\''Arrow'\'')'
        (null)-WARNING **: Failed to load shared library 'libarrow-glib.so.900' referenced by the typelib:
           /lib64/libjemalloc.so.2: cannot allocate memory in static TLS block
    
    This is caused because system jemalloc isn't built with
    --disable-initial-exec-tls.
    
    See also:
    
    * https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=951704
    * https://github.com/jemalloc/jemalloc/issues/1237
    
    Authored-by: Sutou Kouhei <ko...@clear-code.com>
    Signed-off-by: Krisztián Szűcs <sz...@gmail.com>
---
 dev/release/verify-apt.sh                          |  2 ++
 dev/release/verify-yum.sh                          | 39 +++++++++++++++++++---
 .../apache-arrow/apt/debian-bookworm/Dockerfile    |  1 -
 .../apache-arrow/apt/debian-bullseye/Dockerfile    |  1 -
 .../apache-arrow/apt/ubuntu-bionic/Dockerfile      |  1 -
 .../apache-arrow/apt/ubuntu-focal/Dockerfile       |  1 -
 .../apache-arrow/apt/ubuntu-jammy/Dockerfile       |  1 -
 .../linux-packages/apache-arrow/debian/control.in  |  2 --
 .../apache-arrow/debian/libarrow-dev.install       |  1 -
 dev/tasks/linux-packages/apache-arrow/debian/rules |  3 +-
 .../apache-arrow/yum/almalinux-8/Dockerfile        |  1 -
 .../apache-arrow/yum/almalinux-9/Dockerfile        |  1 -
 .../apache-arrow/yum/amazon-linux-2/Dockerfile     |  1 -
 .../linux-packages/apache-arrow/yum/arrow.spec.in  |  5 ---
 .../apache-arrow/yum/centos-7/Dockerfile           |  1 -
 .../apache-arrow/yum/centos-8-stream/Dockerfile    |  1 -
 .../apache-arrow/yum/centos-9-stream/Dockerfile    |  1 -
 17 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
index 846d9c7ec3..121abdbc6e 100755
--- a/dev/release/verify-apt.sh
+++ b/dev/release/verify-apt.sh
@@ -158,6 +158,8 @@ echo "::endgroup::"
 
 
 echo "::group::Test Apache Arrow GLib"
+export G_DEBUG=fatal-warnings
+
 ${APT_INSTALL} libarrow-glib-dev=${package_version}
 ${APT_INSTALL} libarrow-glib-doc=${package_version}
 
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index 889dddd9d1..93bbcfe136 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -46,12 +46,14 @@ repository_version="${distribution_version}"
 
 cmake_package=cmake
 cmake_command=cmake
+have_arrow_libs=no
 have_flight=yes
 have_gandiva=yes
 have_glib=yes
 have_parquet=yes
 have_python=yes
-have_arrow_libs=no
+have_ruby=yes
+ruby_devel_packages=(ruby-devel)
 install_command="dnf install -y --enablerepo=crb"
 uninstall_command="dnf remove -y"
 clean_command="dnf clean"
@@ -63,6 +65,7 @@ case "${distribution}-${distribution_version}" in
   almalinux-8)
     distribution_prefix="almalinux"
     have_arrow_libs=yes
+    ruby_devel_packages+=(redhat-rpm-config)
     install_command="dnf install -y --enablerepo=powertools"
     info_command="dnf info --enablerepo=powertools"
     ;;
@@ -73,12 +76,13 @@ case "${distribution}-${distribution_version}" in
     distribution_prefix="amazon-linux"
     cmake_package=cmake3
     cmake_command=cmake3
-    have_flight=no
-    have_gandiva=no
-    have_python=no
     if [ "$(arch)" != "aarch64" ]; then
       have_arrow_libs=yes
     fi
+    have_flight=no
+    have_gandiva=no
+    have_python=no
+    have_ruby=no
     install_command="yum install -y"
     uninstall_command="yum remove -y"
     clean_command="yum clean"
@@ -89,10 +93,11 @@ case "${distribution}-${distribution_version}" in
     distribution_prefix="centos"
     cmake_package=cmake3
     cmake_command=cmake3
+    have_arrow_libs=yes
     have_flight=no
     have_gandiva=no
     have_python=no
-    have_arrow_libs=yes
+    have_ruby=no
     install_command="yum install -y"
     uninstall_command="yum remove -y"
     clean_command="yum clean"
@@ -101,6 +106,7 @@ case "${distribution}-${distribution_version}" in
   centos-8)
     distribution_prefix="centos"
     repository_version+="-stream"
+    ruby_devel_packages+=(redhat-rpm-config)
     install_command="dnf install -y --enablerepo=powertools"
     info_command="dnf info --enablerepo=powertools"
     ;;
@@ -206,6 +212,8 @@ echo "::endgroup::"
 
 if [ "${have_glib}" = "yes" ]; then
   echo "::group::Test Apache Arrow GLib"
+  export G_DEBUG=fatal-warnings
+
   ${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
 
@@ -215,6 +223,12 @@ if [ "${have_glib}" = "yes" ]; then
   valac --pkg arrow-glib --pkg posix build.vala
   ./build
   popd
+
+  if [ "${have_ruby}" = "yes" ]; then
+    ${install_command} "${ruby_devel_packages[@]}"
+    gem install gobject-introspection
+    ruby -r gi -e "p GI.load('Arrow')"
+  fi
   echo "::endgroup::"
 fi
 
@@ -222,11 +236,17 @@ if [ "${have_flight}" = "yes" ]; then
   echo "::group::Test Apache Arrow Flight"
   ${install_command} --enablerepo=epel arrow-flight-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-flight-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('ArrowFlight')"
+  fi
   echo "::endgroup::"
 
   echo "::group::Test Apache Arrow Flight SQL"
   ${install_command} --enablerepo=epel arrow-flight-sql-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-flight-sql-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('ArrowFlightSQL')"
+  fi
   echo "::endgroup::"
 fi
 
@@ -240,6 +260,9 @@ echo "::group::Test Plasma"
 if [ "${have_glib}" = "yes" ]; then
   ${install_command} --enablerepo=epel plasma-glib-devel-${package_version}
   ${install_command} --enablerepo=epel plasma-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('Plasma')"
+  fi
 else
   ${install_command} --enablerepo=epel plasma-devel-${package_version}
 fi
@@ -250,6 +273,9 @@ if [ "${have_gandiva}" = "yes" ]; then
   if [ "${have_glib}" = "yes" ]; then
     ${install_command} --enablerepo=epel gandiva-glib-devel-${package_version}
     ${install_command} --enablerepo=epel gandiva-glib-doc-${package_version}
+    if [ "${have_ruby}" = "yes" ]; then
+      ruby -r gi -e "p GI.load('Gandiva')"
+    fi
   else
     ${install_command} --enablerepo=epel gandiva-devel-${package_version}
   fi
@@ -261,6 +287,9 @@ if [ "${have_parquet}" = "yes" ]; then
   if [ "${have_glib}" = "yes" ]; then
     ${install_command} --enablerepo=epel parquet-glib-devel-${package_version}
     ${install_command} --enablerepo=epel parquet-glib-doc-${package_version}
+    if [ "${have_ruby}" = "yes" ]; then
+      ruby -r gi -e "p GI.load('Parquet')"
+    fi
   else
     ${install_command} --enablerepo=epel parquet-devel-${package_version}
   fi
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
index 42a7327614..152ac08a7c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
@@ -53,7 +53,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
index 1562f1d662..440a07f09c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
@@ -53,7 +53,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
index 64ebb3b72d..b897fa73a3 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -48,7 +48,6 @@ RUN \
     libglib2.0-doc \
     libgoogle-glog-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libre2-dev \
     libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index 9efc560268..5abee7c2e0 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -50,7 +50,6 @@ RUN \
     libgmock-dev \
     libgoogle-glog-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libre2-dev \
     libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
index e6f657533f..7e95c37642 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -51,7 +51,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
index 8de71bda75..e0a6378828 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control.in
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -18,7 +18,6 @@ Build-Depends:
   libgoogle-glog-dev,
 @USE_SYSTEM_GRPC@  libgrpc++-dev,
   libgtest-dev,
-  libjemalloc-dev,
   liblz4-dev,
 @USE_SYSTEM_GRPC@  libprotoc-dev,
 @USE_SYSTEM_GRPC@  libprotobuf-dev,
@@ -147,7 +146,6 @@ Depends:
   libbz2-dev,
 @USE_SYSTEM_C_ARES@  libc-ares-dev,
 @USE_SYSTEM_GRPC@  libgrpc++-dev,
-  libjemalloc-dev,
   liblz4-dev,
   libre2-dev,
   libsnappy-dev,
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
index b55c76d692..40a834ced4 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
@@ -4,7 +4,6 @@ usr/lib/*/cmake/arrow/ArrowOptions.cmake
 usr/lib/*/cmake/arrow/ArrowTargets*.cmake
 usr/lib/*/cmake/arrow/Find*Alt.cmake
 usr/lib/*/cmake/arrow/FindArrow.cmake
-usr/lib/*/cmake/arrow/Findjemalloc.cmake
 usr/lib/*/cmake/arrow/Find[BTuz]*.cmake
 usr/lib/*/cmake/arrow/arrow-config.cmake
 usr/lib/*/libarrow.a
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
index 9178111eba..2f3b694d68 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -62,8 +62,7 @@ override_dh_auto_configure:
 	  -DCUDAToolkit_ROOT=/usr				\
 	  -DPARQUET_REQUIRE_ENCRYPTION=ON			\
 	  -DPythonInterp_FIND_VERSION=ON			\
-	  -DPythonInterp_FIND_VERSION_MAJOR=3			\
-	  -Djemalloc_SOURCE=SYSTEM
+	  -DPythonInterp_FIND_VERSION_MAJOR=3
 
 override_dh_auto_build:
 	dh_auto_build				\
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
index 0a37bab4b1..a75fd022bd 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
index 8561779f65..df63f7a1a7 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
index fa1ded66bc..68d604d64e 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
@@ -37,7 +37,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     lz4-devel \
     make \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 78f974f971..098342bacd 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -105,7 +105,6 @@ BuildRequires:	git
 %if %{use_glog}
 BuildRequires:	glog-devel
 %endif
-BuildRequires:	jemalloc-devel
 %if %{use_gcs}
 BuildRequires:	json-devel
 %endif
@@ -194,7 +193,6 @@ cd cpp
   -DPythonInterp_FIND_VERSION=ON \
   -DPythonInterp_FIND_VERSION_MAJOR=3 \
 %endif
-  -Djemalloc_SOURCE=SYSTEM \
   -G"Unix Makefiles"
 %arrow_cmake_build
 cd -
@@ -243,7 +241,6 @@ Requires:	gflags
 %if %{use_glog}
 Requires:	glog
 %endif
-Requires:	jemalloc
 %if %{have_zstd}
 Requires:	libzstd
 %endif
@@ -275,7 +272,6 @@ Requires:	bzip2-devel
 %if %{use_flight}
 Requires:	c-ares-devel
 %endif
-Requires:	jemalloc-devel
 %if %{use_gcs}
 Requires:	json-devel
 %endif
@@ -323,7 +319,6 @@ Libraries and header files for Apache Arrow C++.
 %if %{use_flight}
 %{_libdir}/cmake/arrow/Findc-aresAlt.cmake
 %endif
-%{_libdir}/cmake/arrow/Findjemalloc.cmake
 %{_libdir}/cmake/arrow/Findlz4Alt.cmake
 %if %{have_re2}
 %{_libdir}/cmake/arrow/Findre2Alt.cmake
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
index 93d4a7812b..9c93e2f240 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libzstd-devel \
     lz4-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
index ca9e44d409..b29cc4565b 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
index 2bc036432c..38e6ae4531 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \