You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2018/04/01 17:32:48 UTC
[airavata] branch develop updated: Supporting kafka logging for
helix tasks
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new df058b5 Supporting kafka logging for helix tasks
df058b5 is described below
commit df058b5a8419ad7c39dd579460a13a40ebe33948
Author: dimuthu <di...@gmail.com>
AuthorDate: Sun Apr 1 13:32:39 2018 -0400
Supporting kafka logging for helix tasks
---
modules/airavata-helix/helix-spectator/pom.xml | 10 +++++
.../helix-spectator/src/main/resources/logback.xml | 43 ++++++++++++++++++++++
2 files changed, 53 insertions(+)
diff --git a/modules/airavata-helix/helix-spectator/pom.xml b/modules/airavata-helix/helix-spectator/pom.xml
index 213f4e6..fd68ee3 100644
--- a/modules/airavata-helix/helix-spectator/pom.xml
+++ b/modules/airavata-helix/helix-spectator/pom.xml
@@ -91,5 +91,15 @@
<artifactId>log4j-over-slf4j</artifactId>
<version>1.7.25</version>
</dependency>
+ <dependency>
+ <groupId>com.github.danielwegener</groupId>
+ <artifactId>logback-kafka-appender</artifactId>
+ <version>0.2.0-RC1</version>
+ </dependency>
+ <dependency>
+ <groupId>net.logstash.logback</groupId>
+ <artifactId>logstash-logback-encoder</artifactId>
+ <version>5.0</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/modules/airavata-helix/helix-spectator/src/main/resources/logback.xml b/modules/airavata-helix/helix-spectator/src/main/resources/logback.xml
index ef38b65..78c89ee 100644
--- a/modules/airavata-helix/helix-spectator/src/main/resources/logback.xml
+++ b/modules/airavata-helix/helix-spectator/src/main/resources/logback.xml
@@ -41,12 +41,55 @@
</rollingPolicy>
</appender>
+ <appender name="KAFKA_FAILED_LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/kafka-failed.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %X - %m%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/kafka-failed.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+
+ <appender name="RELAXED-KAFKA-APPENDER" class="com.github.danielwegener.logback.kafka.KafkaAppender">
+ <encoder class="net.logstash.logback.encoder.LogstashEncoder">
+ <pattern>%d [%t] %-5p %c{30} %X - %m%n</pattern>
+ </encoder>
+ <topic>airavata-logs</topic>
+ <!-- we don't care how the log messages will be partitioned -->
+ <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy" />
+
+ <!-- use async delivery. the application threads are not blocked by logging -->
+ <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />
+
+ <!-- each <producerConfig> translates to regular kafka-client config (format: key=value) -->
+ <!-- producer configs are documented here: https://kafka.apache.org/documentation.html#newproducerconfigs -->
+ <!-- bootstrap.servers is the only mandatory producerConfig -->
+ <producerConfig>bootstrap.servers=192.168.99.103:9092</producerConfig>
+ <!-- don't wait for a broker to ack the reception of a batch. -->
+ <producerConfig>acks=0</producerConfig>
+ <!-- wait up to 1000ms and collect log messages before sending them as a batch -->
+ <producerConfig>linger.ms=1000</producerConfig>
+ <!-- even if the producer buffer runs full, do not block the application but start to drop messages -->
+ <producerConfig>max.block.ms=0</producerConfig>
+ <!-- define a client-id that you use to identify yourself against the kafka broker -->
+ <producerConfig>client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed</producerConfig>
+
+ <!-- there is no fallback <appender-ref>. If this appender cannot deliver, it will drop its messages. -->
+ <appender-ref ref="KAFKA_FAILED_LOGFILE"/>
+ </appender>
+
<logger name="ch.qos.logback" level="WARN"/>
<logger name="org.apache.helix" level="WARN"/>
<logger name="org.apache.zookeeper" level="ERROR"/>
<logger name="org.apache.airavata" level="INFO"/>
<logger name="org.hibernate" level="ERROR"/>
<root level="INFO">
+ <appender-ref ref="RELAXED-KAFKA-APPENDER"/>
<appender-ref ref="CONSOLE"/>
<appender-ref ref="LOGFILE"/>
</root>
--
To stop receiving notification emails like this one, please contact
dimuthuupe@apache.org.