You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ji...@apache.org on 2016/10/20 03:10:27 UTC
incubator-eagle git commit: [EAGLE-644] optimize some default
configurations for JPM
Repository: incubator-eagle
Updated Branches:
refs/heads/master 44259aeca -> c3bdf46ca
[EAGLE-644] optimize some default configurations for JPM
Author: wujinhu <wu...@126.com>
Closes #535 from wujinhu/EAGLE-642.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/c3bdf46c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/c3bdf46c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/c3bdf46c
Branch: refs/heads/master
Commit: c3bdf46ca290cd9b36a09d79943d680af2ece5f4
Parents: 44259ae
Author: wujinhu <wu...@126.com>
Authored: Thu Oct 20 11:10:19 2016 +0800
Committer: wujinhu <wu...@126.com>
Committed: Thu Oct 20 11:10:19 2016 +0800
----------------------------------------------------------------------
...he.eagle.jpm.aggregation.AggregationApplicationProvider.xml | 2 +-
...he.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml | 2 +-
.../org/apache/eagle/jpm/mr/running/parser/MRJobParser.java | 6 ++----
...he.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml | 6 +++---
4 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
index 2b328cc..aa35d11 100644
--- a/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-aggregation/src/main/resources/META-INF/providers/org.apache.eagle.jpm.aggregation.AggregationApplicationProvider.xml
@@ -25,7 +25,7 @@
<name>workers</name>
<displayName>Worker Number</displayName>
<description>the number of storm workers will be used</description>
- <value>4</value>
+ <value>2</value>
</property>
<property>
<name>stormConfig.mrHistoryAggregationSpoutTasks</name>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index 77a3709..ee09d50 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -26,7 +26,7 @@
<name>workers</name>
<displayName>Worker Number</displayName>
<description>the number of storm workers will be used</description>
- <value>4</value>
+ <value>2</value>
</property>
<property>
<name>stormConfig.mrHistoryJobSpoutTasks</name>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 797bf21..edcb9f7 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -74,11 +74,11 @@ public class MRJobParser implements Runnable {
private MRRunningJobManager runningJobManager;
private ParserStatus parserStatus;
private ResourceFetcher rmResourceFetcher;
- private boolean first;
private Set<String> finishedTaskIds;
private List<String> configKeys;
private MRRunningJobConfig.EndpointConfig endpointConfig;
private static final int TOP_BOTTOM_TASKS_BY_ELASPED_TIME = 10;
+ private static final int FLUSH_TASKS_EVERY_TIME = 5;
static {
OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
@@ -106,7 +106,6 @@ public class MRJobParser implements Runnable {
this.runningJobManager = runningJobManager;
this.parserStatus = ParserStatus.FINISHED;
this.rmResourceFetcher = rmResourceFetcher;
- this.first = true;
this.finishedTaskIds = new HashSet<>();
this.configKeys = configKeys;
}
@@ -150,11 +149,10 @@ public class MRJobParser implements Runnable {
List<Function<String, Boolean>> functions = new ArrayList<>();
functions.add(fetchJobConfig);
functions.add(fetchJobCounters);
- if (!this.first) { //do not fetch these info below for the first time
+ if ((int)(Math.random() * 10) % FLUSH_TASKS_EVERY_TIME == 0) {
functions.add(fetchTasks);
}
- this.first = false;
for (String jobId : mrJobEntityMap.keySet()) {
for (Function<String, Boolean> function : functions) {
int i = 0;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/c3bdf46c/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index 4063b3a..d57629a 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -25,7 +25,7 @@
<name>workers</name>
<displayName>Worker Number</displayName>
<description>the number of storm workers will be used</description>
- <value>4</value>
+ <value>2</value>
</property>
<property>
<name>stormConfig.mrRunningJobFetchSpoutTasks</name>
@@ -37,7 +37,7 @@
<name>stormConfig.mrRunningJobParseBoltTasks</name>
<displayName>Parse Task Number</displayName>
<description>number of tasks to parse map reduce running jobs got from resource manager</description>
- <value>5</value>
+ <value>6</value>
</property>
<property>
@@ -58,7 +58,7 @@
<name>endpointConfig.parseJobThreadPoolSize</name>
<displayName>Parse Job ThreadPool Size in Each Parse Task</displayName>
<description>parse job thread pool size in each parse task</description>
- <value>5</value>
+ <value>2</value>
</property>
<property>
<name>endpointConfig.rmUrls</name>