[incubator-pinot] branch master updated: Fix log4j2 for Quickstart. (#6919)

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 9035323  Fix log4j2 for Quickstart. (#6919)
9035323 is described below

commit 903532355ddc4b61dcddb6a1f632f2ba2670531b
Author: Mayank Shrivastava <ma...@apache.org>
AuthorDate: Thu May 13 18:21:37 2021 -0700

    Fix log4j2 for Quickstart. (#6919)
    
    In QuickStart, we start all Pinot components in a single JVM. And the existing
    log4j2 config for it directs package specific logs to different components.
    This is broken because packages can be shared across components (eg pinot-core).
    
    During debugging recent problems reported in OSS, we found that realtime
    consumption would stop in Pinot, but no errors/exceptions would appear in
    the logs. This was due to the fact that the exceptions were thrown by packages
    that were not part of any component's log, and were missed.
    
    In this PR we address the problem as:
    - We create a single log file with log level `info` for quick-start, as it
      starts all components in single JVM.
    - We set the log level to error for console.
---
 .../src/main/resources/conf/quickstart-log4j2.xml  | 76 +++++-----------------
 1 file changed, 17 insertions(+), 59 deletions(-)

diff --git a/pinot-tools/src/main/resources/conf/quickstart-log4j2.xml b/pinot-tools/src/main/resources/conf/quickstart-log4j2.xml
index 8a1879e..594ea41 100644
--- a/pinot-tools/src/main/resources/conf/quickstart-log4j2.xml
+++ b/pinot-tools/src/main/resources/conf/quickstart-log4j2.xml
@@ -20,71 +20,29 @@
 
 -->
 <Configuration>
+  <Properties>
+    <Property name="LOG_ROOT">logs</Property>
+    <Property name="LOG_PATTERN">%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Property>
+    <Property name="PINOT_COMPONENT">quickstart</Property>
+  </Properties>
   <Appenders>
-    <Console name="console" target="SYSTEM_OUT"/>
-    <RandomAccessFile name="controllerLog" fileName="pinotController.log" immediateFlush="false">
-      <PatternLayout>
-        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
-      </PatternLayout>
-    </RandomAccessFile>
-    <RandomAccessFile name="brokerLog" fileName="pinotBroker.log" immediateFlush="false">
-      <PatternLayout>
-        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
-      </PatternLayout>
-    </RandomAccessFile>
-    <RandomAccessFile name="serverLog" fileName="pinotServer.log" immediateFlush="false">
-      <PatternLayout>
-        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
-      </PatternLayout>
-    </RandomAccessFile>
-    <RandomAccessFile name="minionLog" fileName="pinotMinion.log" immediateFlush="false">
-      <PatternLayout>
-        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
-      </PatternLayout>
-    </RandomAccessFile>
-
+    <Console name="console" target="SYSTEM_OUT" />
+    <RollingFile name="pinotLog" fileName="${env:LOG_ROOT}/pinot-${env:PINOT_COMPONENT}.log"
+                 filePattern="${env:LOG_ROOT}/pinot-${env:PINOT_COMPONENT}.log.%i.gz" immediateFlush="false">
+      <PatternLayout pattern="${env:LOG_PATTERN}"/>
+      <Policies>
+        <SizeBasedTriggeringPolicy size="19500KB" />
+      </Policies>
+      <DefaultRolloverStrategy max="10"/>
+    </RollingFile>
   </Appenders>
   <Loggers>
-    <Root level="error" additivity="false">
-      <AppenderRef ref="console"/>
+    <Root level="info" additivity="false">
+      <AppenderRef ref="pinotLog"/>
     </Root>
-    <Logger name="org.apache.pinot" level="error" additivity="false"/>
-    <Logger name="org.apache.pinot.tools.admin" level="info" additivity="false">
+    <Logger name="org.apache.pinot.tools" level="error" additivity="false">
       <AppenderRef ref="console"/>
     </Logger>
-
-    <!-- Direct controller package log to the controller log file -->
-    <AsyncLogger name="org.apache.pinot.controller" level="info" additivity="false">
-      <AppenderRef ref="controllerLog"/>
-    </AsyncLogger>
-
-    <!-- Direct broker package log to the broker log file -->
-    <AsyncLogger name="org.apache.pinot.broker" level="info" additivity="false">
-      <AppenderRef ref="brokerLog"/>
-    </AsyncLogger>
-
-    <!-- Including server related package log to the server log file -->
-    <AsyncLogger name="org.apache.pinot.server" level="info" additivity="false">
-      <AppenderRef ref="serverLog"/>
-    </AsyncLogger>
-    <AsyncLogger name="org.apache.pinot.core.plan" level="info" additivity="false">
-      <AppenderRef ref="serverLog"/>
-    </AsyncLogger>
-    <AsyncLogger name="org.apache.pinot.core.realtime" level="info" additivity="false">
-      <AppenderRef ref="serverLog"/>
-    </AsyncLogger>
-    <AsyncLogger name="org.apache.pinot.core.query" level="info" additivity="false">
-      <AppenderRef ref="serverLog"/>
-    </AsyncLogger>
-
-    <!-- Direct minion package log to the broker log file -->
-    <AsyncLogger name="org.apache.pinot.minion" level="info" additivity="false">
-      <AppenderRef ref="minionLog"/>
-    </AsyncLogger>
-
     <AsyncLogger name="org.reflections" level="error" additivity="false"/>
-    <AsyncLogger name="org.apache.pinot.spi.plugin" level="error" additivity="false">
-      <AppenderRef ref="console"/>
-    </AsyncLogger>
   </Loggers>
 </Configuration>

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org