You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/11/07 22:06:00 UTC
[3/3] ambari git commit: AMBARI-22300. Log Feeder: allow applying
grok filter on default fields (like type, path, host etc.) (oleewere)
AMBARI-22300. Log Feeder: allow applying grok filter on default fields (like type, path, host etc.) (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da98efc4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da98efc4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da98efc4
Branch: refs/heads/branch-2.6
Commit: da98efc4a6fa4e941ab1b997cb73f1e410de936c
Parents: 289b251
Author: Oliver Szabo <ol...@gmail.com>
Authored: Wed Oct 25 00:45:10 2017 +0200
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Tue Nov 7 22:53:34 2017 +0100
----------------------------------------------------------------------
.../ambari/logfeeder/filter/FilterGrok.java | 3 +++
.../apache/ambari/logfeeder/input/Input.java | 7 +++++++
.../ambari/logfeeder/output/OutputManager.java | 16 +-------------
.../ambari/logfeeder/util/LogFeederUtil.java | 22 ++++++++++++++++++++
4 files changed, 33 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
index deff1b2..49d7e76 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
@@ -185,6 +185,7 @@ public class FilterGrok extends Filter {
if (strBuff != null) {
Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>());
try {
+ LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false);
applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr);
} finally {
strBuff = null;
@@ -204,6 +205,7 @@ public class FilterGrok extends Filter {
} else {
savedInputMarker = inputMarker;
Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>());
+ LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false);
applyMessage(inputStr, jsonObj, null);
}
}
@@ -212,6 +214,7 @@ public class FilterGrok extends Filter {
public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException {
if (sourceField != null) {
savedInputMarker = inputMarker;
+ LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false);
applyMessage((String) jsonObj.get(sourceField), jsonObj, null);
if (removeSourceField) {
jsonObj.remove(sourceField);
http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
index 6f6bb6d..96320e9 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
@@ -83,6 +83,8 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable {
private boolean multiFolder = false;
private Map<String, List<File>> folderMap;
private Map<String, InputFile> inputChildMap = new HashMap<>(); // TODO: weird it has this relationship
+ private boolean initDefaultFields = false;
+
protected MetricData readBytesMetric = new MetricData(getReadBytesMetricName(), false);
protected String getReadBytesMetricName() {
@@ -96,6 +98,7 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable {
detachIntervalMin = getIntValue("detach_interval_min", DEFAULT_DETACH_INTERVAL_MIN * 60);
detachTimeMin = getIntValue("detach_time_min", DEFAULT_DETACH_TIME_MIN * 60);
pathUpdateIntervalMin = getIntValue("path_update_interval_min", DEFAULT_LOG_PATH_UPDATE_INTERVAL_MIN * 60);
+ initDefaultFields = getBooleanValue("init_default_fields", false);
if (typeValue != null) {
// Explicitly add type and value to field list
contextFields.put("type", typeValue);
@@ -440,4 +443,8 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable {
public void setThread(Thread thread) {
this.thread = thread;
}
+
+ public boolean isInitDefaultFields() {
+ return initDefaultFields;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
index 8bf6ae3..c851f04 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
@@ -85,21 +85,7 @@ public class OutputManager {
// TODO: Ideally most of the overrides should be configurable
- if (jsonObj.get("type") == null) {
- jsonObj.put("type", input.getStringValue("type"));
- }
- if (jsonObj.get("path") == null && input.getFilePath() != null) {
- jsonObj.put("path", input.getFilePath());
- }
- if (jsonObj.get("path") == null && input.getStringValue("path") != null) {
- jsonObj.put("path", input.getStringValue("path"));
- }
- if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) {
- jsonObj.put("host", LogFeederUtil.hostName);
- }
- if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
- jsonObj.put("ip", LogFeederUtil.ipAddress);
- }
+ LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, true);
if (jsonObj.get("level") == null) {
jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
index 73cf449..86287cd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
@@ -33,6 +33,8 @@ import java.util.Properties;
import org.apache.ambari.logfeeder.LogFeeder;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
@@ -339,4 +341,24 @@ public class LogFeederUtil {
}
return logfeederTempDir;
}
+
+ public static void fillMapWithFieldDefaults(Map<String, Object> jsonObj, InputMarker inputMarker, boolean force) {
+ if (inputMarker != null && inputMarker.input != null && (force || inputMarker.input.isInitDefaultFields())) {
+ if (jsonObj.get("type") == null) {
+ jsonObj.put("type", inputMarker.input.getStringValue("type"));
+ }
+ if (jsonObj.get("path") == null && inputMarker.input.getFilePath() != null) {
+ jsonObj.put("path", inputMarker.input.getFilePath());
+ }
+ if (jsonObj.get("path") == null && inputMarker.input.getStringValue("path") != null) {
+ jsonObj.put("path", inputMarker.input.getStringValue("path"));
+ }
+ if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) {
+ jsonObj.put("host", LogFeederUtil.hostName);
+ }
+ if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
+ jsonObj.put("ip", LogFeederUtil.ipAddress);
+ }
+ }
+ }
}