You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/07/23 08:46:02 UTC
[ambari] branch trunk updated: AMBARI-24329. Log Feeder define
default log levels per component. (#1837)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1070b65 AMBARI-24329. Log Feeder define default log levels per component. (#1837)
1070b65 is described below
commit 1070b65c2c2479a5687a5a70a9443e154d627ec2
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Mon Jul 23 10:45:57 2018 +0200
AMBARI-24329. Log Feeder define default log levels per component. (#1837)
---
.../api/model/inputconfig/InputDescriptor.java | 3 +++
.../model/inputconfig/impl/InputDescriptorImpl.java | 20 ++++++++++++++++++++
.../loglevelfilter/LogLevelFilterHandler.java | 20 ++++++++++----------
.../ambari/logfeeder/output/OutputManagerImpl.java | 17 +++++++++++++++--
.../ambari/logfeeder/output/OutputManagerTest.java | 6 ++++--
.../ambari/logsearch/model/common/LSServerInput.java | 9 +++++++++
.../shipper-conf/input.config-logsearch-docker.json | 5 ++++-
7 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
index 71f3cd1..7c00a6b 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logsearch.config.api.model.inputconfig;
+import java.util.List;
import java.util.Map;
public interface InputDescriptor {
@@ -53,4 +54,6 @@ public interface InputDescriptor {
String getGroup();
Boolean isInitDefaultFields();
+
+ List<String> getDefaultLogLevels();
}
diff --git a/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
index 3a04ebc..d175c58 100644
--- a/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logsearch.config.json.model.inputconfig.impl;
+import java.util.List;
import java.util.Map;
import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
@@ -200,6 +201,16 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
@SerializedName("init_default_fields")
private Boolean initDefaultFields;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/default_log_levels",
+ type = "list of strings",
+ description = "Use these as default log levels for the input - overrides the global default log levels.",
+ examples = {"default_log_levels: [\"INFO\", \"WARN\"]"}
+ )
+ @Expose
+ @SerializedName("default_log_levels")
+ private List<String> defaultLogLevels;
+
public String getType() {
return type;
}
@@ -329,4 +340,13 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
public void setInitDefaultFields(Boolean initDefaultFields) {
this.initDefaultFields = initDefaultFields;
}
+
+ @Override
+ public List<String> getDefaultLogLevels() {
+ return defaultLogLevels;
+ }
+
+ public void setDefaultLogLevels(List<String> defaultLogLevels) {
+ this.defaultLogLevels = defaultLogLevels;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
index 977e537..8c35d56 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
@@ -98,29 +98,29 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
return filters;
}
- public boolean isAllowed(String hostName, String logId, String level) {
+ public boolean isAllowed(String hostName, String logId, String level, List<String> defaultLogLevels) {
if (!logFeederProps.isLogLevelFilterEnabled()) {
return true;
}
- LogLevelFilter logFilter = findLogFilter(logId);
+ LogLevelFilter logFilter = findLogFilter(logId, defaultLogLevels);
List<String> allowedLevels = getAllowedLevels(hostName, logFilter);
return allowedLevels.isEmpty() || allowedLevels.contains(level);
}
- public boolean isAllowed(String jsonBlock, InputMarker inputMarker) {
+ public boolean isAllowed(String jsonBlock, InputMarker inputMarker, List<String> defaultLogLevels) {
if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) {
return DEFAULT_VALUE;
}
Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
- return isAllowed(jsonObj, inputMarker);
+ return isAllowed(jsonObj, inputMarker, defaultLogLevels);
}
- public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker) {
+ public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker, List<String> defaultLogLevels) {
if ("audit".equals(inputMarker.getInput().getInputDescriptor().getRowtype()))
return true;
- boolean isAllowed = applyFilter(jsonObj);
+ boolean isAllowed = applyFilter(jsonObj, defaultLogLevels);
if (!isAllowed) {
LOG.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj));
}
@@ -128,7 +128,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
}
- public boolean applyFilter(Map<String, Object> jsonObj) {
+ public boolean applyFilter(Map<String, Object> jsonObj, List<String> defaultLogLevels) {
if (MapUtils.isEmpty(jsonObj)) {
LOG.warn("Output jsonobj is empty");
return DEFAULT_VALUE;
@@ -138,13 +138,13 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) && org.apache.commons.lang3.StringUtils.isNotBlank(logId) && org.apache.commons.lang3.StringUtils.isNotBlank(level)) {
- return isAllowed(hostName, logId, level);
+ return isAllowed(hostName, logId, level, defaultLogLevels);
} else {
return DEFAULT_VALUE;
}
}
- private synchronized LogLevelFilter findLogFilter(String logId) {
+ private synchronized LogLevelFilter findLogFilter(String logId, List<String> defaultLogLevels) {
LogLevelFilter logFilter = filters.get(logId);
if (logFilter != null) {
return logFilter;
@@ -153,7 +153,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor {
LOG.info("Filter is not present for log " + logId + ", creating default filter");
LogLevelFilter defaultFilter = new LogLevelFilter();
defaultFilter.setLabel(logId);
- defaultFilter.setDefaultLevels(logFeederProps.getIncludeDefaultLogLevels());
+ defaultFilter.setDefaultLevels(defaultLogLevels);
try {
config.getLogLevelFilterManager().createLogLevelFilter(logFeederProps.getClusterName(), logId, defaultFilter);
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
index 612573b..09951cc 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
@@ -31,6 +31,7 @@ import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -147,7 +148,8 @@ public class OutputManagerImpl extends OutputManager {
jsonObj.put("message_md5", "" + Hashing.md5().hashBytes(logMessage.getBytes()).asLong());
}
}
- if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker)
+ List<String> defaultLogLevels = getDefaultLogLevels(input);
+ if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker, defaultLogLevels)
&& !outputLineFilter.apply(jsonObj, inputMarker.getInput())) {
List<? extends Output> outputList = input.getOutputList();
for (Output output : outputList) {
@@ -160,6 +162,16 @@ public class OutputManagerImpl extends OutputManager {
}
}
+ private List<String> getDefaultLogLevels(Input input) {
+ List<String> defaultLogLevels = logFeederProps.getIncludeDefaultLogLevels();
+ List<String> overrideDefaultLogLevels = input.getInputDescriptor().getDefaultLogLevels();
+ if (CollectionUtils.isNotEmpty(overrideDefaultLogLevels)) {
+ return overrideDefaultLogLevels;
+ } else {
+ return defaultLogLevels;
+ }
+ }
+
@SuppressWarnings("unchecked")
private String truncateLongLogMessage(Map<String, Object> jsonObj, Input input, String logMessage) {
if (logMessage != null && logMessage.getBytes().length > MAX_OUTPUT_SIZE) {
@@ -181,7 +193,8 @@ public class OutputManagerImpl extends OutputManager {
}
public void write(String jsonBlock, InputMarker inputMarker) {
- if (logLevelFilterHandler.isAllowed(jsonBlock, inputMarker)) {
+ List<String> defaultLogLevels = getDefaultLogLevels(inputMarker.getInput());
+ if (logLevelFilterHandler.isAllowed(jsonBlock, inputMarker, defaultLogLevels)) {
List<? extends Output> outputList = inputMarker.getInput().getOutputList();
for (Output output : outputList) {
try {
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
index 1a651ab..9536cf8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
@@ -110,7 +110,7 @@ public class OutputManagerTest {
expect(mockInput.isUseEventMD5()).andReturn(false).anyTimes();
expect(mockInput.isGenEventMD5()).andReturn(false).anyTimes();
expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes();
- expect(mockFilter.isAllowed(jsonObj, inputMarker)).andReturn(true).anyTimes();
+ expect(mockFilter.isAllowed(jsonObj, inputMarker, null)).andReturn(true).anyTimes();
expect(mockInput.getCache()).andReturn(null);
expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
@@ -121,6 +121,7 @@ public class OutputManagerTest {
replay(output1, output2, output3, mockFilter, mockInput);
OutputManagerImpl manager = new OutputManagerImpl();
+ manager.setLogFeederProps(new LogFeederProps());
manager.setLogLevelFilterHandler(mockFilter);
manager.add(output1);
manager.add(output2);
@@ -146,7 +147,7 @@ public class OutputManagerTest {
LogLevelFilterHandler mockFilter = strictMock(LogLevelFilterHandler.class);
expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes();
- expect(mockFilter.isAllowed(jsonString, inputMarker)).andReturn(true).anyTimes();
+ expect(mockFilter.isAllowed(jsonString, inputMarker, null)).andReturn(true).anyTimes();
expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
output1.write(jsonString, inputMarker); expectLastCall();
@@ -157,6 +158,7 @@ public class OutputManagerTest {
OutputManagerImpl manager = new OutputManagerImpl();
manager.setLogLevelFilterHandler(mockFilter);
+ manager.setLogFeederProps(new LogFeederProps());
manager.add(output1);
manager.add(output2);
manager.add(output3);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
index af28f17..272f443 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logsearch.model.common;
+import java.util.List;
import java.util.Map;
import javax.validation.constraints.NotNull;
@@ -73,6 +74,9 @@ public abstract class LSServerInput {
@JsonProperty("init_default_fields")
private Boolean initDefaultFields;
+
+ @JsonProperty("default_log_levels")
+ private List<String> defaultLogLevels;
public LSServerInput() {}
@@ -92,6 +96,7 @@ public abstract class LSServerInput {
this.cacheDedupInterval = inputDescriptor.getCacheDedupInterval();
this.isEnabled = inputDescriptor.isEnabled();
this.initDefaultFields = inputDescriptor.isInitDefaultFields();
+ this.defaultLogLevels = inputDescriptor.getDefaultLogLevels();
}
public String getType() {
@@ -153,4 +158,8 @@ public abstract class LSServerInput {
public Boolean getInitDefaultFields() {
return initDefaultFields;
}
+
+ public List<String> getDefaultLogLevels() {
+ return defaultLogLevels;
+ }
}
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json
index a420960..04a3adc 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json
@@ -3,7 +3,10 @@
{
"type": "logsearch_server",
"rowtype": "service",
- "docker": "true"
+ "docker": "true",
+ "default_log_levels" : [
+ "FATAL", "ERROR", "WARN", "INFO", "DEBUG"
+ ]
}
],
"filter": [