You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2022/09/12 22:38:45 UTC
[pulsar] branch master updated: [improve][broker] Make log flushing configurable in Pulsar broker (#17591)

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new e79642c86ae [improve][broker] Make log flushing configurable in Pulsar broker (#17591)
e79642c86ae is described below

commit e79642c86ae4553e5699be7f3772ef8a865bee65
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Sep 13 01:38:31 2022 +0300

    [improve][broker] Make log flushing configurable in Pulsar broker (#17591)
    
    - by default Pulsar won't flush log statements to disk after each log line. This is problematic while investigating some issues since the broker might crash without the last log lines.
---
 bin/pulsar                  | 2 ++
 bin/pulsar-admin-common.cmd | 2 ++
 bin/pulsar-admin-common.sh  | 2 ++
 bin/pulsar-perf             | 2 ++
 bin/pulsar-perf.cmd         | 2 ++
 conf/log4j2.yaml            | 4 +++-
 6 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/bin/pulsar b/bin/pulsar
index 52676818fce..8292193a7d9 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -320,12 +320,14 @@ PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
 if [ ! -d "$PULSAR_LOG_DIR" ]; then
   mkdir -p "$PULSAR_LOG_DIR"
 fi
+PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
 
 #Configure log configuration system properties
 OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
 OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
 OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+OPTS="$OPTS -Dpulsar.log.immediateFlush=$PULSAR_LOG_IMMEDIATE_FLUSH"
 OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
 # Configure log4j2 to disable servlet webapp detection so that Garbage free logging can be used
 OPTS="$OPTS -Dlog4j2.is.webapp=false"
diff --git a/bin/pulsar-admin-common.cmd b/bin/pulsar-admin-common.cmd
index b518734fa0b..c52bc1389f6 100644
--- a/bin/pulsar-admin-common.cmd
+++ b/bin/pulsar-admin-common.cmd
@@ -71,11 +71,13 @@ if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=RoutingAppender"
 if "%PULSAR_LOG_LEVEL%" == "" set "PULSAR_LOG_LEVEL=info"
 if "%PULSAR_LOG_ROOT_LEVEL%" == "" set "PULSAR_LOG_ROOT_LEVEL=%PULSAR_LOG_LEVEL%"
 if "%PULSAR_ROUTING_APPENDER_DEFAULT%" == "" set "PULSAR_ROUTING_APPENDER_DEFAULT=Console"
+if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false"
 
 set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%"
 set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
 set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
 set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
+set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
 set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%"
 
 :error
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index 7d4b0d861bf..8223ac5b3bf 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -108,10 +108,12 @@ PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
 PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
 PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"}
 PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
+PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
 
 #Configure log configuration system properties
 OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
 OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
 OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+OPTS="$OPTS -Dpulsar.log.immediateFlush=$PULSAR_LOG_IMMEDIATE_FLUSH"
 OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT"
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index 40e3f6587c1..ad22d5be39f 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -142,11 +142,13 @@ PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
 PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"}
 PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
 PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"}
+PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
 
 #Configure log configuration system properties
 OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
 OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
 OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
+OPTS="$OPTS -Dpulsar.log.immediateFlush=$PULSAR_LOG_IMMEDIATE_FLUSH"
 OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
 OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"
 
diff --git a/bin/pulsar-perf.cmd b/bin/pulsar-perf.cmd
index 1a6acfbf6de..cf6c25b77e5 100644
--- a/bin/pulsar-perf.cmd
+++ b/bin/pulsar-perf.cmd
@@ -63,12 +63,14 @@ if "%PULSAR_LOG_FILE%" == "" set "PULSAR_LOG_FILE=pulsar-perftest.log"
 if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=Console"
 if "%PULSAR_LOG_LEVEL%" == "" set "PULSAR_LOG_LEVEL=info"
 if "%PULSAR_LOG_ROOT_LEVEL%" == "" set "PULSAR_LOG_ROOT_LEVEL=%PULSAR_LOG_LEVEL%"
+if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false"
 
 
 set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%"
 set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
 set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
 set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
+set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
 
 set "COMMAND=%1"
 
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index 71698d72067..fff2e063ebd 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -30,6 +30,8 @@ Configuration:
         value: "logs"
       - name: "pulsar.log.file"
         value: "pulsar.log"
+      - name: "pulsar.log.immediateFlush"
+        value: "false"
       - name: "pulsar.log.appender"
         value: "RoutingAppender"
       - name: "pulsar.log.root.level"
@@ -61,7 +63,7 @@ Configuration:
       name: RollingFile
       fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}"
       filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz"
-      immediateFlush: false
+      immediateFlush: ${sys:pulsar.log.immediateFlush}
       PatternLayout:
         Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n"
       Policies: