You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/08/23 04:55:32 UTC
[incubator-iotdb] branch log_tool created (now 406cbbd)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch log_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 406cbbd add a framework
This branch includes the following new commits:
new 406cbbd add a framework
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add a framework
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch log_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 406cbbd1d7e1d5f4fd1870b43744505b7e20cc30
Author: jt <jt...@163.com>
AuthorDate: Fri Aug 23 12:45:36 2019 +0800
add a framework
---
server/pom.xml | 6 +
.../resources/tools/logAnalyze/default.log.pattern | 7 +
.../logAnalyze/plans/flushTimeConsumption.plan | 16 ++
.../apache/iotdb/db/tools/logvisual/LogEntry.java | 85 ++++++++
.../apache/iotdb/db/tools/logvisual/LogFilter.java | 114 +++++++++++
.../apache/iotdb/db/tools/logvisual/LogParser.java | 15 ++
.../iotdb/db/tools/logvisual/LogVisualizer.java | 213 +++++++++++++++++++++
.../iotdb/db/tools/logvisual/PatternLogParser.java | 110 +++++++++++
.../iotdb/db/tools/logvisual/VisualizePlan.java | 144 ++++++++++++++
.../logvisual/exceptions/AnalyzeException.java | 19 ++
.../exceptions/NoLogFileLoadedException.java | 8 +
.../logvisual/exceptions/NoSuchPlanException.java | 8 +
.../exceptions/UnmatchedContentException.java | 8 +
13 files changed, 753 insertions(+)
diff --git a/server/pom.xml b/server/pom.xml
index 9055d28..13f6ebb 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -85,6 +85,12 @@
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
+ <!-- https://mvnrepository.com/artifact/org.jfree/jfreechart -->
+ <dependency>
+ <groupId>org.jfree</groupId>
+ <artifactId>jfreechart</artifactId>
+ <version>1.5.0</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/server/src/assembly/resources/tools/logAnalyze/default.log.pattern b/server/src/assembly/resources/tools/logAnalyze/default.log.pattern
new file mode 100644
index 0000000..fccbd91
--- /dev/null
+++ b/server/src/assembly/resources/tools/logAnalyze/default.log.pattern
@@ -0,0 +1,7 @@
+pattern=([^\\[]*)(\\[.*])(\\s\\w+\\s)([^:]*:\\d+)(\\s-\\s)(.*)
+date_index=1
+thread_name_index=2
+level_index=3
+code_location_index=4
+content_index=6
+date_pattern=yyyy-MM-dd hh:mm:ss,SSS
diff --git a/server/src/assembly/resources/tools/logAnalyze/plans/flushTimeConsumption.plan b/server/src/assembly/resources/tools/logAnalyze/plans/flushTimeConsumption.plan
new file mode 100644
index 0000000..f18e0d6
--- /dev/null
+++ b/server/src/assembly/resources/tools/logAnalyze/plans/flushTimeConsumption.plan
@@ -0,0 +1,16 @@
+name=flushTimeConsumption
+content_pattern=Storage group (.*) memtable (.*) flushing a memtable has finished! Time consumption: (\\d+)ms
+
+# if you add more, use comma to separate
+measurement_positions=3
+legends=Time
+tag_positions=1
+
+# min_level=info
+# thread_name_white_list=pool-1-IoTDB-JDBC-CLient-thread-5
+# class_name_white_list=org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor
+# line_num_white_list=392
+
+date_pattern=yyyy-MM-dd hh:mm:ss
+start_date=2019-08-21 09:00:00
+end_date=2019-08-22 09:00:00
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogEntry.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogEntry.java
new file mode 100644
index 0000000..7146113
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogEntry.java
@@ -0,0 +1,85 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+public class LogEntry {
+
+ private static List<String> DEFAULT_TAG = Collections.EMPTY_LIST;
+
+ private Date date;
+ private String threadName;
+ private LogLevel logLevel;
+ private CodeLocation codeLocation;
+ private String logContent;
+
+ private List<String> tags = DEFAULT_TAG;
+ private List<Double> measurements;
+
+ LogEntry(Date date, String threadName,
+ LogLevel logLevel, CodeLocation codeLocation, String logContent) {
+ this.date = date;
+ this.threadName = threadName;
+ this.logLevel = logLevel;
+ this.codeLocation = codeLocation;
+ this.logContent = logContent;
+ }
+
+ public Date getDate() {
+ return date;
+ }
+
+ public String getThreadName() {
+ return threadName;
+ }
+
+ public LogLevel getLogLevel() {
+ return logLevel;
+ }
+ public CodeLocation getCodeLocation() {
+ return codeLocation;
+ }
+
+ public String getLogContent() {
+ return logContent;
+ }
+
+ public List<String> getTags() {
+ return tags;
+ }
+
+ public void setTags(List<String> tags) {
+ this.tags = tags;
+ }
+
+ public List<Double> getMeasurements() {
+ return measurements;
+ }
+
+ public void setMeasurements(List<Double> measurements) {
+ this.measurements = measurements;
+ }
+
+ enum LogLevel {
+ DEBUG, INFO, WARN, ERROR
+ }
+
+ static class CodeLocation {
+ private String className;
+ private int lineNum;
+
+ CodeLocation(String className, int lineNum) {
+ this.className = className;
+ this.lineNum = lineNum;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public int getLineNum() {
+ return lineNum;
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogFilter.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogFilter.java
new file mode 100644
index 0000000..054fa6e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogFilter.java
@@ -0,0 +1,114 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import static org.apache.iotdb.db.tools.logvisual.LogFilter.FilterProperties.*;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.DATE_PATTERN;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+import org.apache.iotdb.db.tools.logvisual.LogEntry.LogLevel;
+
+public class LogFilter {
+ // optional, only logs with levels equal to or higher than this will be analyzed
+ private LogLevel minLevel = LogLevel.DEBUG;
+ // optional, only threads, classes, lines in the lists are analyzed. When unset, all logs will
+ // analyzed. comma-separated
+ private List<String> threadNameWhiteList;
+ private List<String> classNameWhiteList;
+ private List<Integer> lineNumWhiteList;
+ // optional, only time ranges within the interval will be analyzed
+ // if startDate or endDate is set, datePattern must be set too
+ private DateFormat datePartten;
+ private Date startDate = new Date(Long.MIN_VALUE);
+ private Date endDate = new Date(Long.MAX_VALUE);
+
+ public LogFilter(Properties properties) throws IOException {
+ minLevel = LogLevel.valueOf(properties.getProperty(MIN_LEVEL.getPropertyName(), minLevel.name()));
+
+ String threadNameWhiteListStr = properties.getProperty(THREAD_NAME_WHITE_LIST.getPropertyName
+ ());
+ if (threadNameWhiteListStr != null) {
+ threadNameWhiteList = Arrays.asList(threadNameWhiteListStr.trim().split(","));
+ }
+
+ String classNameWhiteListStr = properties.getProperty(CLASS_NAME_WHITE_LIST.getPropertyName());
+ if (classNameWhiteListStr != null) {
+ classNameWhiteList = Arrays.asList(classNameWhiteListStr.trim().split(","));
+ }
+
+ String lineNumWhiteListStr = properties.getProperty(LINE_NUM_WHITE_LIST.getPropertyName());
+ if (lineNumWhiteListStr != null) {
+ String[] lineNumWhiteListStrs = lineNumWhiteListStr.trim().split(",");
+ lineNumWhiteList = new ArrayList<>();
+ for (int i = 0; i < lineNumWhiteListStrs.length; i++) {
+ lineNumWhiteList.add(Integer.parseInt(lineNumWhiteListStrs[i]));
+ }
+ }
+
+ String datePatternStr = properties.getProperty(DATE_PATTERN.getPropertyName());
+ if (datePatternStr != null) {
+ this.datePartten = new SimpleDateFormat(datePatternStr.trim());
+ String startDateStr = properties.getProperty(START_DATE.getPropertyName());
+ if (startDateStr != null) {
+ try {
+ startDate = datePartten.parse(startDateStr.trim());
+ } catch (ParseException e) {
+ throw new IOException(e);
+ }
+ }
+
+ String endDatePattern = properties.getProperty(END_DATE.getPropertyName());
+ if (startDateStr != null) {
+ try {
+ endDate = datePartten.parse(endDatePattern.trim());
+ } catch (ParseException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+ }
+
+ public FilterFeedBack filter(LogEntry entry) {
+ if (entry.getLogLevel().ordinal() < minLevel.ordinal() ||
+ (threadNameWhiteList != null && !threadNameWhiteList.contains(entry.getThreadName())) ||
+ (classNameWhiteList != null && !classNameWhiteList.contains(entry.getCodeLocation()
+ .getClassName())) ||
+ (lineNumWhiteList != null && !lineNumWhiteList.contains(entry.getCodeLocation().getLineNum
+ ())) ||
+ (startDate != null && entry.getDate().before(startDate))) {
+ return FilterFeedBack.REJECT;
+ }
+
+ if (endDate != null && entry.getDate().after(endDate)) {
+ return FilterFeedBack.BEYOND_END_TIME;
+ }
+ return FilterFeedBack.OK;
+ }
+
+ enum FilterProperties {
+ MIN_LEVEL("min_level"), THREAD_NAME_WHITE_LIST("thread_name_white_list"), CLASS_NAME_WHITE_LIST(
+ "class_name_white_list"),
+ LINE_NUM_WHITE_LIST("line_num_white_list"), START_DATE("start_date"), END_DATE("end_date"),
+ DATE_PATTERN("date_pattern");
+
+ private String propertyName;
+
+ FilterProperties(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+ }
+
+ enum FilterFeedBack {
+ OK, REJECT, BEYOND_END_TIME
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogParser.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogParser.java
new file mode 100644
index 0000000..2675085
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogParser.java
@@ -0,0 +1,15 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import java.io.IOException;
+
+public interface LogParser {
+
+ /**
+ * return the next LogEntry or null if there is no more logs.
+ */
+ LogEntry next() throws IOException;
+
+ void close() throws IOException;
+
+ void reset() throws IOException;
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogVisualizer.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogVisualizer.java
new file mode 100644
index 0000000..bf9b660
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/LogVisualizer.java
@@ -0,0 +1,213 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import java.awt.Graphics;
+import java.awt.image.BufferedImage;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import javax.swing.JFrame;
+import javax.swing.JPanel;
+import org.apache.iotdb.db.tools.logvisual.exceptions.AnalyzeException;
+import org.apache.iotdb.db.tools.logvisual.exceptions.NoLogFileLoadedException;
+import org.apache.iotdb.db.tools.logvisual.exceptions.NoSuchPlanException;
+import org.apache.iotdb.db.tools.logvisual.exceptions.UnmatchedContentException;
+import org.jfree.chart.ChartFactory;
+import org.jfree.chart.JFreeChart;
+import org.jfree.chart.plot.XYPlot;
+import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer;
+import org.jfree.data.time.Millisecond;
+import org.jfree.data.time.TimeSeries;
+import org.jfree.data.time.TimeSeriesCollection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogVisualizer {
+
+ private static final Logger logger = LoggerFactory.getLogger(LogVisualizer.class);
+ private static final String STR_HAPPENED_INSTANCE = "HappenedInstance";
+
+ private LogParser logParser;
+ private List<LogEntry> logCache = new ArrayList<>();
+
+ private Map<String, VisualizePlan> plans = new HashMap<>();
+ private Map<List<String>, List<LogEntry>> logGroups = new HashMap<>();
+
+ private void clearLogs() {
+ logCache.clear();
+ logGroups.clear();
+ }
+
+ public void loadLogParser(String propertyFilePath, String logFilePath)
+ throws IOException {
+ Properties properties = new Properties();
+ try (FileInputStream inputStream = new FileInputStream(propertyFilePath);
+ BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream)) {
+ properties.load(bufferedInputStream);
+ }
+ logParser = new PatternLogParser(properties, logFilePath);
+ }
+
+ public void close() throws IOException {
+ logParser.close();
+ logParser = null;
+ }
+
+ public void loadPlan(String planFilePath) throws IOException {
+ try (FileInputStream reader = new FileInputStream(planFilePath);
+ BufferedInputStream bufferedInputStream = new BufferedInputStream(reader)) {
+ Properties properties = new Properties();
+ properties.load(bufferedInputStream);
+ VisualizePlan plan = new VisualizePlan(properties);
+ plans.put(plan.getName(), plan);
+ }
+ }
+
+ public Collection<String> listPlans() {
+ return plans.keySet();
+ }
+
+ public void executePlan(String planName) throws AnalyzeException {
+ VisualizePlan plan = plans.get(planName);
+ if (plan == null) {
+ throw new NoSuchPlanException(planName);
+ }
+ if (logParser == null) {
+ throw new NoLogFileLoadedException();
+ }
+ try {
+ logParser.reset();
+ } catch (IOException e) {
+ throw new AnalyzeException(e);
+ }
+ collectLogs(plan);
+ groupLogs();
+ Map<String,TimeSeriesCollection> taggedTimeSeries = createTimeSeries(plan);
+ List<JFreeChart> charts = drawCharts(taggedTimeSeries, plan);
+ showCharts(charts.subList(0,10));
+ }
+
+ private void collectLogs(VisualizePlan plan) throws AnalyzeException {
+ LogFilter logFilter = plan.getLogFilter();
+ clearLogs();
+ try {
+ LogEntry logEntry;
+ readLogs:
+ while ((logEntry = logParser.next()) != null) {
+ try {
+ plan.parseContents(logEntry);
+ } catch (UnmatchedContentException e) {
+ continue;
+ }
+
+ switch (logFilter.filter(logEntry)) {
+ case BEYOND_END_TIME:
+ break readLogs;
+ case REJECT:
+ break;
+ case OK:
+ logCache.add(logEntry);
+ }
+ }
+ } catch (IOException e) {
+ throw new AnalyzeException(e);
+ }
+ logger.info("Collected {} logs", logCache.size());
+ }
+
+ private void groupLogs() {
+ for (LogEntry logEntry : logCache) {
+ logGroups.computeIfAbsent(logEntry.getTags(), tag -> new ArrayList<>()).add(logEntry);
+ }
+ logger.info("Found {} different tags", logGroups.size());
+ }
+
+ private Map<String, TimeSeriesCollection> createTimeSeries(VisualizePlan plan) {
+ Map<String, TimeSeriesCollection> ret = new HashMap<>();
+ for (Entry<List<String>, List<LogEntry>> entry : logGroups.entrySet()) {
+ List<String> tags = entry.getKey();
+ List<LogEntry> logs = entry.getValue();
+ String concatenatedTag;
+ if (tags.isEmpty()) {
+ concatenatedTag = plan.getName();
+ } else {
+ StringBuilder builder = new StringBuilder(plan.getName() + "-" + tags.get(0));
+ for (int i = 1; i < tags.size(); i++) {
+ builder.append(",").append(tags.get(i));
+ }
+ builder.append(" ");
+ concatenatedTag = builder.toString();
+ }
+ TimeSeriesCollection tagTimeseries = new TimeSeriesCollection();
+ if (plan.getMeasurementPositions() != null) {
+ String[] legends = plan.getLegends();
+ for (String legend : legends) {
+ tagTimeseries.addSeries(new TimeSeries(concatenatedTag + legend));
+ }
+ for (LogEntry logEntry : logs) {
+ List<Double> values = logEntry.getMeasurements();
+ for (int i = 0; i < values.size(); i++) {
+ tagTimeseries.getSeries(i).addOrUpdate(new Millisecond(logEntry.getDate()), values.get(i));
+ }
+ }
+ } else {
+ TimeSeries happenedInstance = new TimeSeries("HappenedInstance");
+ for (LogEntry logEntry : logs) {
+ happenedInstance.add(new Millisecond(logEntry.getDate()), 1.0);
+ }
+ }
+ ret.put(concatenatedTag, tagTimeseries);
+ }
+ return ret;
+ }
+
+ private List<JFreeChart> drawCharts(Map<String, TimeSeriesCollection> taggedTimeSeries, VisualizePlan plan) {
+ List<JFreeChart> charts = new ArrayList<>();
+ for (Entry<String, TimeSeriesCollection> entry : taggedTimeSeries.entrySet()) {
+ String tag = entry.getKey();
+ TimeSeriesCollection timeSeriesList = entry.getValue();
+ if (plan.getMeasurementPositions() != null) {
+ // a real-valued timeseries, draw a curve
+ JFreeChart chart = ChartFactory.createTimeSeriesChart(tag, "time", "value", timeSeriesList);
+ XYPlot xyPlot = chart.getXYPlot();
+ ((XYLineAndShapeRenderer) xyPlot.getRenderer()).setDefaultShapesFilled(true);
+ charts.add(chart);
+ } else {
+ // a binary timeseries, draw a scatter plot
+ JFreeChart chart = ChartFactory.createScatterPlot(tag, "time", "", timeSeriesList);
+ charts.add(chart);
+ }
+ }
+ return charts;
+ }
+
+ private void showCharts(List<JFreeChart> charts) {
+ for (JFreeChart chart : charts) {
+ BufferedImage image = chart.createBufferedImage(800, 600);
+ JFrame frame = new JFrame();
+ frame.add(new JPanel() {
+ @Override
+ protected void paintComponent(Graphics g) {
+ g.drawImage(image, 0, 0, null);
+ }
+ });
+ frame.setVisible(true);
+ frame.setSize(880, 660);
+ }
+ }
+
+ public static void main(String[] args) throws IOException, AnalyzeException {
+ LogVisualizer visualizer = new LogVisualizer();
+ visualizer.loadLogParser("E:\\codestore\\incubator-iotdb\\server\\src\\assembly\\resources"
+ + "\\tools\\logAnalyze\\default.log.pattern",
+ "C:\\Users\\admin\\Desktop\\logs\\log-all-2019-08-21.0.log");
+ visualizer.loadPlan("E:\\codestore\\incubator-iotdb\\server\\src\\assembly\\resources\\tools\\logAnalyze\\plans\\flushTimeConsumption.plan");
+ visualizer.executePlan("flushTimeConsumption");
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/PatternLogParser.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/PatternLogParser.java
new file mode 100644
index 0000000..93d793c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/PatternLogParser.java
@@ -0,0 +1,110 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.CODE_LOCATION_INDEX;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.CONTENT_INDEX;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.DATE_INDEX;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.DATE_PATTERN;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.LEVEL_INDEX;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.PATTERN;
+import static org.apache.iotdb.db.tools.logvisual.PatternLogParser.PatternProperties.THREAD_NAME_INDEX;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.iotdb.db.tools.logvisual.LogEntry.CodeLocation;
+import org.apache.iotdb.db.tools.logvisual.LogEntry.LogLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PatternLogParser implements LogParser{
+ private static final Logger logger = LoggerFactory.getLogger(LogParser.class);
+
+ private Pattern pattern;
+ private int dateIndex;
+ private int threadNameIndex;
+ private int levelIndex;
+ private int codeLocationIndex;
+ private int contentIndex;
+
+ private String logFilePath;
+ private BufferedReader reader;
+ private DateFormat dateFormat;
+
+ PatternLogParser(Properties properties, String logFilePath) throws IOException {
+ this.pattern = Pattern.compile(properties.getProperty(PATTERN.getPropertyName()));
+ this.dateIndex = Integer.parseInt(properties.getProperty(DATE_INDEX.getPropertyName()));
+ this.threadNameIndex = Integer.parseInt(properties.getProperty(THREAD_NAME_INDEX
+ .getPropertyName()));
+ this.levelIndex = Integer.parseInt(properties.getProperty(LEVEL_INDEX.getPropertyName()));
+ this.codeLocationIndex = Integer.parseInt(properties.getProperty(CODE_LOCATION_INDEX
+ .getPropertyName()));
+ this.contentIndex = Integer.parseInt(properties.getProperty(CONTENT_INDEX.getPropertyName()));
+ this.dateFormat = new SimpleDateFormat(properties.getProperty(DATE_PATTERN.getPropertyName()));
+ this.logFilePath = logFilePath;
+ }
+
+ @Override
+ public LogEntry next() throws IOException {
+ String line = reader.readLine();
+ if (line == null) {
+ return null;
+ }
+
+ Matcher matcher = pattern.matcher(line);
+ if (!matcher.matches()) {
+ logger.error("Unrecognizable log: {}", line);
+ return null;
+ }
+ Date date;
+ try {
+ date = dateFormat.parse(matcher.group(dateIndex));
+ } catch (ParseException e) {
+ logger.error("Incorrect time format in {}", e);
+ return null;
+ }
+ String threadName = matcher.group(threadNameIndex).trim();
+ LogLevel logLevel = LogLevel.valueOf(matcher.group(levelIndex).trim());
+ String[] codeLocationStr = matcher.group(codeLocationIndex).split(":");
+ CodeLocation codeLocation = new CodeLocation(codeLocationStr[0].trim(), Integer.parseInt
+ (codeLocationStr[1]));
+ String content = matcher.group(contentIndex).trim();
+ return new LogEntry(date, threadName, logLevel, codeLocation, content);
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (reader != null) {
+ reader.close();
+ }
+ reader = new BufferedReader(new InputStreamReader(new FileInputStream(logFilePath)));
+ }
+
+ enum PatternProperties {
+ PATTERN("pattern"), DATE_INDEX("date_index"), THREAD_NAME_INDEX("thread_name_index"),
+ LEVEL_INDEX("level_index"), CODE_LOCATION_INDEX("code_location_index"), CONTENT_INDEX
+ ("content_index"), DATE_PATTERN("date_pattern");
+
+ private String propertyName;
+
+ PatternProperties(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/VisualizePlan.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/VisualizePlan.java
new file mode 100644
index 0000000..431f305
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/VisualizePlan.java
@@ -0,0 +1,144 @@
+package org.apache.iotdb.db.tools.logvisual;
+
+import static org.apache.iotdb.db.tools.logvisual.VisualizePlan.PlanProperties.*;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.iotdb.db.tools.logvisual.exceptions.UnmatchedContentException;
+
+public class VisualizePlan {
+ // optional, this will be used as the title of the figure.
+ private String name;
+
+ // required, a regex that will capture the logs to be analyzed, the interesting values
+ // (measurements and group-by tags) should be surrounded with bracelets e.g.:
+ // contentPattern = location:(.*) temperature:(.*) pressure:(.*)
+ private Pattern contentPattern;
+ // the following 3 are optional, if not set, then it means this plan only cares whether this
+ // event happens or not and draws a scatter plot, other wise it will capture the given
+ // measurements and draw curves. Only numeric measurements are supported currently.
+ // if one of first 2 is set, the other must be set
+ // comma-separated
+ // e.g.:
+ // measurementPositions = 2,3
+ // legends = temperature,pressure
+ // tagPositions = 1
+ // then the logs will be grouped-by their locations and for each group, there will be two
+ // curves describing temperature and pressure respectively
+ private int[] measurementPositions;
+ private String[] legends;
+ private int[] tagPositions;
+
+ private LogFilter logFilter;
+
+ public VisualizePlan(Properties properties) throws IOException {
+ this.name = properties.getProperty(NAME.getPropertyName(), "untitled");
+ String patternStr = properties.getProperty(CONTENT_PATTERN.getPropertyName());
+ if (patternStr == null) {
+ throw new IOException("Bad plan, content pattern unset");
+ }
+ this.contentPattern = Pattern.compile(patternStr);
+
+ String measurementPositionsStr = properties.getProperty(MEASUREMENT_POSITIONS.getPropertyName());
+ if (measurementPositionsStr != null) {
+ String[] measurePosStrs = measurementPositionsStr.split(",");
+ measurementPositions = new int[measurePosStrs.length];
+ for (int i = 0; i < measurementPositions.length; i++) {
+ measurementPositions[i] = Integer.parseInt(measurePosStrs[i]);
+ }
+ }
+
+ String legendStr = properties.getProperty(LEGENDS.getPropertyName());
+ if (legendStr != null) {
+ legends = legendStr.split(",");
+ }
+
+ String groupByPositionsStr = properties.getProperty(TAG_POSITIONS.getPropertyName());
+ if (groupByPositionsStr != null) {
+ String[] groupByPosStrs = groupByPositionsStr.split(",");
+ tagPositions = new int[groupByPosStrs.length];
+ for (int i = 0; i < tagPositions.length; i++) {
+ tagPositions[i] = Integer.parseInt(groupByPosStrs[i]);
+ }
+ }
+
+ logFilter = new LogFilter(properties);
+ }
+
+ public void parseContents(LogEntry logEntry) throws UnmatchedContentException {
+ Matcher matcher = contentPattern.matcher(logEntry.getLogContent());
+ if (!matcher.matches()) {
+ throw new UnmatchedContentException(logEntry.getLogContent(), contentPattern.pattern());
+ }
+ String[] matchedValues = new String[matcher.groupCount()];
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ matchedValues[i - 1] = matcher.group(i);
+ }
+ if (tagPositions != null) {
+ List<String> tags = new ArrayList<>();
+ for (int pos : tagPositions) {
+ tags.add(matchedValues[pos-1]);
+ }
+ logEntry.setTags(tags);
+ }
+ if (measurementPositions != null) {
+ List<Double> measurements = new ArrayList<>();
+ for (int pos : measurementPositions) {
+ measurements.add(Double.parseDouble(matchedValues[pos-1]));
+ }
+ logEntry.setMeasurements(measurements);
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Pattern getContentPattern() {
+ return contentPattern;
+ }
+
+ public int[] getMeasurementPositions() {
+ return measurementPositions;
+ }
+
+ public String[] getLegends() {
+ return legends;
+ }
+
+ public int[] getTagPositions() {
+ return tagPositions;
+ }
+
+ public LogFilter getLogFilter() {
+ return logFilter;
+ }
+
+ enum PlanProperties {
+ NAME("name"), CONTENT_PATTERN("content_pattern"), MEASUREMENT_POSITIONS(
+ "measurement_positions"),
+ LEGENDS("legends"), TAG_POSITIONS("tag_positions");
+
+ private String propertyName;
+
+ PlanProperties(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ Properties properties = new Properties();
+ properties.load(new FileInputStream("E:\\codestore\\incubator-iotdb\\server\\src\\assembly\\resources\\tools\\logAnalyze\\plans\\flushTimeConsumption.plan"));
+
+ VisualizePlan plan = new VisualizePlan(properties);
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/AnalyzeException.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/AnalyzeException.java
new file mode 100644
index 0000000..0bc72c0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/AnalyzeException.java
@@ -0,0 +1,19 @@
+package org.apache.iotdb.db.tools.logvisual.exceptions;
+
+public class AnalyzeException extends Exception {
+
+ public AnalyzeException() {
+ }
+
+ public AnalyzeException(String message) {
+ super(message);
+ }
+
+ public AnalyzeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AnalyzeException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoLogFileLoadedException.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoLogFileLoadedException.java
new file mode 100644
index 0000000..54338f3
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoLogFileLoadedException.java
@@ -0,0 +1,8 @@
+package org.apache.iotdb.db.tools.logvisual.exceptions;
+
+public class NoLogFileLoadedException extends AnalyzeException {
+
+ public NoLogFileLoadedException() {
+ super("No log file is loaded, please load a log file first");
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoSuchPlanException.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoSuchPlanException.java
new file mode 100644
index 0000000..32c04fc
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/NoSuchPlanException.java
@@ -0,0 +1,8 @@
+package org.apache.iotdb.db.tools.logvisual.exceptions;
+
+public class NoSuchPlanException extends AnalyzeException {
+
+ public NoSuchPlanException(String planName) {
+ super(String.format("No such plan %s", planName));
+ }
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/UnmatchedContentException.java b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/UnmatchedContentException.java
new file mode 100644
index 0000000..9c015a9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/tools/logvisual/exceptions/UnmatchedContentException.java
@@ -0,0 +1,8 @@
+package org.apache.iotdb.db.tools.logvisual.exceptions;
+
+public class UnmatchedContentException extends AnalyzeException {
+
+ public UnmatchedContentException(String content, String pattern) {
+ super(String.format("%s cannot match %s", content, pattern));
+ }
+}
\ No newline at end of file