[incubator-celeborn] branch main updated: [CELEBORN-900][FOLLOWUP] Disable jemalloc in non-docker environment

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 8b7989ad0 [CELEBORN-900][FOLLOWUP] Disable jemalloc in non-docker environment
8b7989ad0 is described below

commit 8b7989ad0cd9e19decdc8b772c92b924426f31a5
Author: sychen <sy...@ctrip.com>
AuthorDate: Mon Sep 11 14:55:10 2023 +0800

    [CELEBORN-900][FOLLOWUP] Disable jemalloc in non-docker environment
    
    ### What changes were proposed in this pull request?
    1. Provide `CELEBORN_PREFER_JEMALLOC` configuration to determine whether to enable jemalloc
    2. Provide `CELEBORN_JEMALLOC_PATH` to configure the jemalloc path, for example, Centos is `/usr/lib64/libjemalloc.so`
    3. Enable jemalloc by default in the docker environment
    
    ### Why are the changes needed?
    Prevent unnecessary WARNING.
    
    https://github.com/apache/incubator-celeborn/pull/1824#discussion_r1319909938
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    local test
    
    Closes #1895 from cxzl25/CELEBORN-900_diable.
    
    Lead-authored-by: sychen <sy...@ctrip.com>
    Co-authored-by: Cheng Pan <pa...@gmail.com>
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
 conf/celeborn-env.sh.template | 5 ++++-
 docker/Dockerfile             | 3 ++-
 sbin/load-celeborn-env.sh     | 7 ++++---
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/conf/celeborn-env.sh.template b/conf/celeborn-env.sh.template
index bada8c172..c7d8eb190 100755
--- a/conf/celeborn-env.sh.template
+++ b/conf/celeborn-env.sh.template
@@ -23,6 +23,8 @@
 # - CELEBORN_WORKER_JAVA_OPTS
 # - CELEBORN_PID_DIR
 # - CELEBORN_LOG_DIR
+# - CELEBORN_PREFER_JEMALLOC, to enable jemalloc memory allocator
+# - CELEBORN_JEMALLOC_PATH, to set jemalloc library path
 
 # Example:
 # CELEBORN_MASTER_MEMORY=2g
@@ -32,4 +34,5 @@
 # CELEBORN_MASTER_JAVA_OPTS="-XX:-PrintGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:gc-master.out -Dio.netty.leakDetectionLevel=advanced"
 # CELEBORN_PID_DIR="$CELEBORN_HOME/pids"
 # CELEBORN_LOG_DIR="$CELEBORN_HOME/logs"
-# DISABLE_JEMALLOC="false"
+# CELEBORN_PREFER_JEMALLOC="true"
+# CELEBORN_JEMALLOC_PATH="/path/to/libjemalloc.so"
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 17a4e0592..d1369436e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -22,7 +22,8 @@ ARG celeborn_uid=10006
 ARG celeborn_gid=10006
 
 ENV CELEBORN_HOME=/opt/celeborn
-ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/busybox:${CELEBORN_HOME}/sbin:${CELEBORN_HOME}/bin
+ENV CELEBORN_PREFER_JEMALLOC=true
+ENV PATH=${PATH}:/opt/busybox:${CELEBORN_HOME}/sbin:${CELEBORN_HOME}/bin
 
 RUN set -ex && \
     apt-get update && \
diff --git a/sbin/load-celeborn-env.sh b/sbin/load-celeborn-env.sh
index 9a36e1f36..7389dc87e 100755
--- a/sbin/load-celeborn-env.sh
+++ b/sbin/load-celeborn-env.sh
@@ -69,10 +69,11 @@ if [ "$CELEBORN_PID_DIR" = "" ]; then
   export CELEBORN_PID_DIR="${CELEBORN_HOME}/pids"
 fi
 
-# jemalloc memory allocator is enabled by default, you may set DISABLE_JEMALLOC to true in celeborn-env.sh if it's not as you wish.
+# The jemalloc memory allocator is disabled by default, but in the docker environment, jemalloc is enabled by default.
+# You can set CELEBORN_PREFER_JEMALLOC to true in celeborn-env.sh and configure the path to jemalloc via CELEBORN_JEMALLOC_PATH.
 maybe_enable_jemalloc() {
-  if [ "${DISABLE_JEMALLOC:-false}" == "false" ]; then
-    JEMALLOC_PATH="/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so"
+  if [ "${CELEBORN_PREFER_JEMALLOC:-false}" == "true" ]; then
+    JEMALLOC_PATH="${CELEBORN_JEMALLOC_PATH:-/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so}"
     JEMALLOC_FALLBACK="/usr/lib/x86_64-linux-gnu/libjemalloc.so"
     if [ -f "$JEMALLOC_PATH" ]; then
       export LD_PRELOAD="$LD_PRELOAD:$JEMALLOC_PATH"