You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/11/22 13:53:05 UTC
[ambari-logsearch] branch master updated: AMBARI-24943. Log Feeder:
log level filter does not have any affect if level field is null (#38)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new f355523 AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null (#38)
f355523 is described below
commit f3555232e931b3a073d448fcd3874f76300830e2
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Thu Nov 22 14:53:01 2018 +0100
AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null (#38)
* AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null
* Remove unused import
---
.../loglevelfilter/LogLevelFilterHandler.java | 8 +--
.../logfeeder/output/OutputLineEnricher.java | 8 ++-
.../ambari/logfeeder/output/OutputManagerImpl.java | 4 +-
.../output/cloud/CloudStorageOutputManager.java | 2 +-
.../shipper-conf/input.config-grafana.json | 67 ++++++++++++++++++++++
docker/test-logs/grafana/grafana.log | 8 +++
6 files changed, 89 insertions(+), 8 deletions(-)
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
index a1780cf..a00e99b 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
@@ -30,7 +30,7 @@ import org.apache.ambari.logsearch.config.zookeeper.LogLevelFilterManagerZK;
import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigZKHelper;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
@@ -133,7 +133,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
}
public boolean isAllowed(String jsonBlock, InputMarker inputMarker, List<String> defaultLogLevels) {
- if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) {
+ if (StringUtils.isEmpty(jsonBlock)) {
return DEFAULT_VALUE;
}
Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
@@ -160,8 +160,8 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
- String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
- if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) && org.apache.commons.lang3.StringUtils.isNotBlank(logId) && org.apache.commons.lang3.StringUtils.isNotBlank(level)) {
+ String level = (String) jsonObj.getOrDefault(LogFeederConstants.SOLR_LEVEL, LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ if (StringUtils.isNotBlank(hostName) && StringUtils.isNotBlank(logId) && StringUtils.isNotBlank(level)) {
return isAllowed(hostName, logId, level, defaultLogLevels);
} else {
return DEFAULT_VALUE;
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
index ff0805d..0b1c5a2 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.output;
import com.google.common.hash.Hashing;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.plugin.common.MetricData;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
@@ -42,7 +43,7 @@ public class OutputLineEnricher {
private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
- public void enrichFields(final Map<String, Object> jsonObj, final InputMarker inputMarker, final MetricData messageTruncateMetric) {
+ public Map<String, Object> enrichFields(final Map<String, Object> jsonObj, final InputMarker inputMarker, final MetricData messageTruncateMetric) {
Input input = inputMarker.getInput();
// Update the block with the context fields
for (Map.Entry<String, String> entry : input.getInputDescriptor().getAddFields().entrySet()) {
@@ -79,12 +80,17 @@ public class OutputLineEnricher {
(Integer) inputMarker.getAllProperties().get("line_number") > 0) {
jsonObj.put("logfile_line_number", inputMarker.getAllProperties().get("line_number"));
}
+ if (!jsonObj.containsKey("level")) {
+ jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ }
if (jsonObj.containsKey("log_message")) {
// TODO: Let's check size only for log_message for now
String logMessage = (String) jsonObj.get("log_message");
logMessage = truncateLongLogMessage(messageTruncateMetric, jsonObj, input, logMessage);
jsonObj.put("message_md5", "" + Hashing.md5().hashBytes(logMessage.getBytes()).asLong());
}
+
+ return jsonObj;
}
@SuppressWarnings("unchecked")
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
index b4c862d..9f40b91 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
@@ -82,10 +82,10 @@ public class OutputManagerImpl extends OutputManager {
@SuppressWarnings("unchecked")
public void write(Map<String, Object> jsonObj, InputMarker inputMarker) {
jsonObj.put("seq_num", docCounter++);
- if (docCounter == Long.MIN_VALUE) {
+ if (docCounter == Long.MAX_VALUE) {
docCounter = 1;
}
- outputLineEnricher.enrichFields(jsonObj, inputMarker, messageTruncateMetric);
+ jsonObj = outputLineEnricher.enrichFields(jsonObj, inputMarker, messageTruncateMetric);
Input input = inputMarker.getInput();
List<String> defaultLogLevels = getDefaultLogLevels(input);
if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker, defaultLogLevels)
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
index 9be30a0..4aca09d 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
@@ -60,7 +60,7 @@ public class CloudStorageOutputManager extends OutputManager {
@Override
public void write(Map<String, Object> jsonObj, InputMarker marker) {
if (useFilters.get()) {
- outputLineEnricher.enrichFields(jsonObj, marker, messageTruncateMetric);
+ jsonObj = outputLineEnricher.enrichFields(jsonObj, marker, messageTruncateMetric);
if (!outputLineFilter.apply(jsonObj, marker.getInput())) {
if (jsonObj.get("id") == null) {
jsonObj.put("id", IdGeneratorHelper.generateUUID(jsonObj, storageOutput.getIdFields()));
diff --git a/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json
new file mode 100644
index 0000000..863050e
--- /dev/null
+++ b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json
@@ -0,0 +1,67 @@
+{
+ "input": [
+ {
+ "type": "ams_grafana",
+ "rowtype": "service",
+ "path": "/root/test-logs/grafana/grafana.log"
+ }
+ ],
+ "filter": [
+ {
+ "filter": "grok",
+ "conditions": {
+ "fields": {
+ "type": [
+ "ams_grafana"
+ ]
+ }
+ },
+ "skipOnError": false,
+ "deepExtract": false,
+ "post_map_values": {
+ "level": [
+ {
+ "map_field_value": {
+ "pre_value": "I",
+ "post_value": "INFO"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "W",
+ "post_value": "WARN"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "D",
+ "post_value": "DEBUG"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "E",
+ "post_value": "ERROR"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "F",
+ "post_value": "FATAL"
+ }
+ }
+ ],
+ "logtime": [
+ {
+ "map_date": {
+ "target_date_pattern": "yyyy/MM/dd HH:mm:ss"
+ }
+ }
+ ]
+ },
+ "log4j_format": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern": "^(%{DATESTAMP:logtime})",
+ "message_pattern": "(?m)^%{DATESTAMP:logtime}%{SPACE}(?:%{DATA:method})%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docker/test-logs/grafana/grafana.log b/docker/test-logs/grafana/grafana.log
new file mode 100644
index 0000000..8d561b9
--- /dev/null
+++ b/docker/test-logs/grafana/grafana.log
@@ -0,0 +1,8 @@
+2018/11/22 10:45:09 [I] Migrator: exec migration id: create index UQE_quota_org_id_user_id_target - v1
+2018/11/22 10:45:09 [I] Created default admin user: admin
+2018/11/22 10:45:09 [I] Listen: http://0.0.0.0:3000
+Thu Nov 22 10:45:10 UTC 2018 pid_file has been written to
+OK
+2018/11/22 10:45:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 1: ambari-metrics
+2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 2: ambari-metrics
+2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 3: ambari-metrics