You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/12/12 13:57:31 UTC
[1/2] ambari git commit: AMBARI-22630. Update the default storm log
search configuration (use wildcards) (oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6-logfeeder-fixes [created] db0a74569
AMBARI-22630. Update the default storm log search configuration (use wildcards) (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e263290
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e263290
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e263290
Branch: refs/heads/branch-2.6-logfeeder-fixes
Commit: 4e2632904277e68c88a8457f2cb00f39364762de
Parents: 5624f2c
Author: Oliver Szabo <ol...@gmail.com>
Authored: Tue Dec 12 00:58:31 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Tue Dec 12 14:51:16 2017 +0100
----------------------------------------------------------------------
.../configuration/storm-logsearch-conf.xml | 98 ++++++++++++++------
1 file changed, 71 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e263290/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
index d485017..ff14aff 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
@@ -69,42 +69,86 @@
{
"type":"storm_worker",
"rowtype":"service",
- "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log"
+ "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/*/*/worker.log",
+ "init_default_fields": "true"
}
],
- "filter":[
- {
- "filter":"grok",
- "conditions":{
- "fields":{
- "type":[
- "storm_drpc",
- "storm_logviewer",
- "storm_nimbus",
- "storm_supervisor",
- "storm_ui",
- "storm_worker"
- ]
+ "filter":[
+ {
+ "filter":"grok",
+ "sort_order": 1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_nimbus",
+ "storm_supervisor",
+ "storm_logviewer",
+ "storm_drpc",
+ "storm_ui",
+ "storm_worker"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}\\s%{GREEDYDATA:thread_name}\\s\\[%{LOGLEVEL:level}\\]\\s%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+ }
+ }
}
},
- "log4j_format":"",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
- "logtime":{
- "map_date":{
- "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
- }
- }
+ {
+ "filter":"grok",
+ "sort_order": 2,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "thread_name",
+ "remove_source_field": "false",
+ "message_pattern":"(Thread\\-[0-9]+\\-[0-9]+\\-%{DATA:sdi_streamline_component_name}\\-executor%{DATA}|%{DATA:thread_name})"
+ },
+ {
+ "filter":"grok",
+ "sort_order": 3,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "path",
+ "remove_source_field": "false",
+ "message_pattern":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/worker\\.log"
+ },
+ {
+ "filter":"grok",
+ "sort_order": 4,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "sdi_storm_topology_id",
+ "remove_source_field": "false",
+ "message_pattern":"(streamline\\-%{DATA:sdi_streamline_topology_id}\\-%{DATA:sdi_streamline_topology_name}\\-[0-9]+\\-[0-9]+)|(%{DATA:sdi_storm_topology_id})"
}
- }
- ]
- }
+ ]
+ }
</value>
<value-attributes>
<type>content</type>
<show-property-name>false</show-property-name>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="true" update="true"/>
</property>
</configuration>
[2/2] ambari git commit: AMBARI-22633. MapDate provides the date
incorrectly when Filter is cloned and used in multi-threads (Jungtaek Lim via
oleewere)
Posted by ol...@apache.org.
AMBARI-22633. MapDate provides the date incorrectly when Filter is cloned and used in multi-threads (Jungtaek Lim via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db0a7456
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db0a7456
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db0a7456
Branch: refs/heads/branch-2.6-logfeeder-fixes
Commit: db0a74569741900457bc14cdd1c36fa5754552cd
Parents: 4e26329
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Dec 12 14:50:39 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Tue Dec 12 14:51:17 2017 +0100
----------------------------------------------------------------------
ambari-logsearch/ambari-logsearch-logfeeder/pom.xml | 5 +++++
.../apache/ambari/logfeeder/mapper/MapperDate.java | 14 +++++++-------
2 files changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db0a7456/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index a06ae90..db2923e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -88,6 +88,11 @@
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.6</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db0a7456/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 6a7fad7..dca3949 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -19,7 +19,6 @@
package org.apache.ambari.logfeeder.mapper;
-import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
@@ -28,15 +27,16 @@ import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
public class MapperDate extends Mapper {
private static final Logger LOG = Logger.getLogger(MapperDate.class);
- private SimpleDateFormat targetDateFormatter = null;
+ private FastDateFormat targetDateFormatter = null;
private boolean isEpoch = false;
- private SimpleDateFormat srcDateFormatter=null;
+ private FastDateFormat srcDateFormatter=null;
@Override
public boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs) {
@@ -61,9 +61,9 @@ public class MapperDate extends Mapper {
return true;
} else {
try {
- targetDateFormatter = new SimpleDateFormat(targetDateFormat);
+ targetDateFormatter = FastDateFormat.getInstance(targetDateFormat);
if (!StringUtils.isEmpty(srcDateFormat)) {
- srcDateFormatter = new SimpleDateFormat(srcDateFormat);
+ srcDateFormatter = FastDateFormat.getInstance(srcDateFormat);
}
return true;
} catch (Throwable ex) {
@@ -86,7 +86,7 @@ public class MapperDate extends Mapper {
if (srcDateFormatter != null) {
Date srcDate = srcDateFormatter.parse(value.toString());
//set year in src_date when src_date does not have year component
- if (!srcDateFormatter.toPattern().contains("yy")) {
+ if (!srcDateFormatter.getPattern().contains("yy")) {
Calendar currentCalendar = Calendar.getInstance();
Calendar logDateCalendar = Calendar.getInstance();
logDateCalendar.setTimeInMillis(srcDate.getTime());
@@ -110,7 +110,7 @@ public class MapperDate extends Mapper {
jsonObj.put(fieldName, value);
} catch (Throwable t) {
LogFeederUtil.logErrorMessageByInterval(this.getClass().getSimpleName() + ":apply", "Error applying date transformation." +
- " isEpoch=" + isEpoch + ", targetateFormat=" + (targetDateFormatter!=null ?targetDateFormatter.toPattern():"")
+ " isEpoch=" + isEpoch + ", targetDateFormat=" + (targetDateFormatter!=null ?targetDateFormatter.getPattern():"")
+ ", value=" + value + ". " + this.toString(), t, LOG, Level.ERROR);
}
}