You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2017/01/23 06:40:01 UTC
eagle git commit: [MINOR] fix small bugs in MR job performance
monitoring
Repository: eagle
Updated Branches:
refs/heads/master eae6e8f11 -> 9381c2d17
[MINOR] fix small bugs in MR job performance monitoring
Author: Zhao, Qingwen <qi...@apache.org>
Closes #789 from qingwen220/minor.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/9381c2d1
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/9381c2d1
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/9381c2d1
Branch: refs/heads/master
Commit: 9381c2d17e3e10a02e9f0f359a05d0728ee17752
Parents: eae6e8f
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Mon Jan 23 14:39:52 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Mon Jan 23 14:39:52 2017 +0800
----------------------------------------------------------------------
.../apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 2 +-
.../jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java | 4 ++++
.../jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java | 3 +--
.../java/org/apache/eagle/jpm/analyzer/publisher/Result.java | 4 +++-
4 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
index e32a37c..57e1765 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
@@ -48,7 +48,7 @@ public class MRJobPerformanceAnalyzer<T extends AnalyzerEntity> implements JobAn
evaluators.add(new JobSuggestionEvaluator(config));
publishers.add(new EagleStorePublisher(config));
- publishers.add(new EmailPublisher(config));
+ //publishers.add(new EmailPublisher(config));
}
@Override
http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
index 103de7a..4007747 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceGCTimeProcessor.java
@@ -20,6 +20,7 @@ package org.apache.eagle.jpm.analyzer.mr.suggestion;
import org.apache.eagle.jpm.analyzer.Processor;
import org.apache.eagle.jpm.analyzer.meta.model.MapReduceAnalyzerEntity;
import org.apache.eagle.jpm.analyzer.publisher.Result;
+import org.apache.eagle.jpm.util.Constants;
import org.apache.eagle.jpm.util.jobcounter.JobCounters;
import java.util.ArrayList;
@@ -45,9 +46,11 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti
long mapGCTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.GC_MILLISECONDS);
long mapCPUTime = context.getJob().getMapCounters().getCounterValue(JobCounters.CounterName.CPU_MILLISECONDS);
+ String message = "%s GC_MILLISECONDS is: %s, and CPU_MILLISECONDS is: %s. ";
if (mapGCTime > mapCPUTime * 0.1) {
setting = String.format("-D%s", MAP_JAVA_OPTS);
optSettings.add(setting);
+ sb.append(String.format(message, Constants.TaskType.MAP, mapGCTime, mapCPUTime));
sb.append("Map GC_TIME_MILLIS took too long. Please increase mapper memory via ").append(setting);
sb.append(", or optimize your mapper class.\n");
}
@@ -58,6 +61,7 @@ public class MapReduceGCTimeProcessor implements Processor<MapReduceAnalyzerEnti
if (reduceGCTime > reduceCPUTime * 0.1) {
setting = String.format("-D%s", REDUCE_JAVA_OPTS);
optSettings.add(setting);
+ sb.append(String.format(message, Constants.TaskType.REDUCE, reduceGCTime, reduceCPUTime));
sb.append("Reduce GC_TIME_MILLIS took too long. Please increase memory for reduce via ").append(setting);
sb.append(", or optimize your reducer class.\n");
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
index 835b382..96be2d5 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/MapReduceSpillProcessor.java
@@ -82,7 +82,7 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit
long heapSize = getMaxHeapSize(context.getJobconf().get(MAP_JAVA_OPTS));
if (heapSize < 3 * minMapSpillMemMB) {
long expectedHeapSizeMB = (minMapSpillMemMB * 3 + 1024) / 1024 * 1024;
- setting = String.format(" -D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB);
+ setting = String.format("-D%s=-Xmx%sM", MAP_JAVA_OPTS, expectedHeapSizeMB);
sb.append(" ").append(setting);
optSettings.add(setting);
}
@@ -90,7 +90,6 @@ public class MapReduceSpillProcessor implements Processor<MapReduceAnalyzerEntit
sb.append(" to avoid spilled records.\n");
}
-
long reduceInputRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.REDUCE_INPUT_RECORDS);
spillRecords = context.getJob().getReduceCounters().getCounterValue(JobCounters.CounterName.SPILLED_RECORDS);
if (reduceInputRecords < spillRecords) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/9381c2d1/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
index a9f5132..7d7442b 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/publisher/Result.java
@@ -63,9 +63,11 @@ public class Result {
}
private void normalizeResult(ProcessorResult processorResult) {
+ String settingList = "";
if (processorResult.getSettings() != null && !processorResult.getSettings().isEmpty()) {
- processorResult.setSettingList(StringUtils.join(processorResult.getSettings(), "\n"));
+ settingList = StringUtils.join(processorResult.getSettings(), "\n");
}
+ processorResult.setSettingList(settingList);
}
/**