You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/02/03 08:56:23 UTC
ambari git commit: AMBARI-19847 Improve Log Feeder simulation to help
scale testing (mgergely)
Repository: ambari
Updated Branches:
refs/heads/trunk 14cd837ea -> 61f8f6d56
AMBARI-19847 Improve Log Feeder simulation to help scale testing (mgergely)
Change-Id: I8a53b3285b919523a929656c10d4d1215169fbab
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61f8f6d5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61f8f6d5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61f8f6d5
Branch: refs/heads/trunk
Commit: 61f8f6d56937996ea7ca2f64b8961e83b72422fd
Parents: 14cd837
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Fri Feb 3 09:56:10 2017 +0100
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Fri Feb 3 09:56:10 2017 +0100
----------------------------------------------------------------------
.../ambari/logfeeder/input/InputSimulate.java | 52 ++++++++++++--------
.../ambari/logfeeder/util/LogFeederUtil.java | 12 +++++
2 files changed, 44 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/61f8f6d5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
index 743be69..be97a52 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
@@ -21,24 +21,25 @@ package org.apache.ambari.logfeeder.input;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.filter.FilterJSON;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.commons.lang3.StringUtils;
import org.apache.solr.common.util.Base64;
+import com.google.common.base.Joiner;
+
public class InputSimulate extends Input {
- private static final String LOG_MESSAGE_PREFIX = "Simulated log message for testing, line";
-
- private static final String LOG_TEXT_PATTERN =
- "{ logtime=\"%d\", level=\"%s\", log_message=\"<LOG_MESSAGE_PATTERN>\"}";
+ private static final String LOG_TEXT_PATTERN = "{ logtime=\"%d\", level=\"%s\", log_message=\"%s\"}";
private static final Map<String, String> typeToFilePath = new HashMap<>();
public static void loadTypeToFilePath(List<Map<String, Object>> inputList) {
@@ -55,16 +56,21 @@ public class InputSimulate extends Input {
private final List<String> types;
private final String level;
- private final String logText;
+ private final int numberOfWords;
+ private final int minLogWords;
+ private final int maxLogWords;
private final long sleepMillis;
public InputSimulate() throws Exception {
this.types = getSimulatedLogTypes();
this.level = LogFeederUtil.getStringProperty("logfeeder.simulate.log_level", "WARN");
- this.logText = getLogText();
+ this.numberOfWords = LogFeederUtil.getIntProperty("logfeeder.simulate.number_of_words", 1000, 50, 1000000);
+ this.minLogWords = LogFeederUtil.getIntProperty("logfeeder.simulate.min_log_words", 5, 1, 10);
+ this.maxLogWords = LogFeederUtil.getIntProperty("logfeeder.simulate.max_log_words", 10, 10, 20);
this.sleepMillis = LogFeederUtil.getIntProperty("logfeeder.simulate.sleep_milliseconds", 10000);
Filter filter = new FilterJSON();
+ filter.loadConfig(Collections.<String, Object> emptyMap());
filter.setInput(this);
addFilter(filter);
}
@@ -80,15 +86,6 @@ public class InputSimulate extends Input {
}
}
- private String getLogText() {
- int logTextSize = LogFeederUtil.getIntProperty("logfeeder.simulate.log_message_size", 100);
- int fillerSize = Math.max(logTextSize - LOG_MESSAGE_PREFIX.length() - 10, 0);
- String filler = StringUtils.repeat("X", fillerSize);
- String logMessagePattern = LOG_MESSAGE_PREFIX + " %08d " + filler;
-
- return LOG_TEXT_PATTERN.replaceAll("<LOG_MESSAGE_PATTERN>", logMessagePattern);
- }
-
@Override
public boolean isReady() {
return true;
@@ -103,12 +100,12 @@ public class InputSimulate extends Input {
while (true) {
String type = imitateRandomLogFile();
+ String line = getLine();
InputMarker marker = getInputMarker(type);
- String line = getLine(marker);
outputLine(line, marker);
- try { Thread.sleep(sleepMillis); } catch(Exception e) {}
+ try { Thread.sleep(sleepMillis); } catch(Exception e) { /* Ignore */ }
}
}
@@ -143,9 +140,24 @@ public class InputSimulate extends Input {
return Base64.byteArrayToBase64(fileKey.getBytes());
}
- private String getLine(InputMarker marker) {
+ private String getLine() {
Date d = new Date();
- return String.format(logText, d.getTime(), level, marker.lineNumber);
+ String logMessage = createLogMessage();
+ return String.format(LOG_TEXT_PATTERN, d.getTime(), level, logMessage);
+ }
+
+ private String createLogMessage() {
+ int logMessageLength = minLogWords + random.nextInt(maxLogWords - minLogWords + 1);
+ Set<Integer> words = new TreeSet<>();
+ List<String> logMessage = new ArrayList<>();
+ while (words.size() < logMessageLength) {
+ int word = random.nextInt(numberOfWords);
+ if (words.add(word)) {
+ logMessage.add(String.format("Word%06d", word));
+ }
+ }
+
+ return Joiner.on(' ').join(logMessage);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/61f8f6d5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
index 5bf600e..73cf449 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
@@ -171,8 +171,20 @@ public class LogFeederUtil {
}
public static int getIntProperty(String key, int defaultValue) {
+ return getIntProperty(key, defaultValue, null, null);
+ }
+
+ public static int getIntProperty(String key, int defaultValue, Integer minValue, Integer maxValue) {
String value = getStringProperty(key);
int retValue = objectToInt(value, defaultValue, ", key=" + key);
+ if (minValue != null && retValue < minValue) {
+ LOG.info("Minimum rule was applied for " + key + ": " + retValue + " < " + minValue);
+ retValue = minValue;
+ }
+ if (maxValue != null && retValue > maxValue) {
+ LOG.info("Maximum rule was applied for " + key + ": " + retValue + " > " + maxValue);
+ retValue = maxValue;
+ }
return retValue;
}