You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ji...@apache.org on 2016/10/25 04:07:59 UTC
incubator-eagle git commit: [EAGLE-679] Fix missing CoprocessorService and MR Running feeder task number limi…
Repository: incubator-eagle
Updated Branches:
refs/heads/master 6b87fb3a7 -> 15f39705d
[EAGLE-679] Fix missing CoprocessorService and MR Running feeder task number limi\u2026
Author: wujinhu <wu...@126.com>
Closes #558 from wujinhu/EAGLE-679.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/15f39705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/15f39705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/15f39705
Branch: refs/heads/master
Commit: 15f39705d0e4c86803d1c552810a3adf2f9b50d6
Parents: 6b87fb3
Author: wujinhu <wu...@126.com>
Authored: Tue Oct 25 12:07:48 2016 +0800
Committer: wujinhu <wu...@126.com>
Committed: Tue Oct 25 12:07:48 2016 +0800
----------------------------------------------------------------------
.../apache/eagle/jpm/mr/running/parser/MRJobParser.java | 11 +++++++++++
eagle-topology-assembly/pom.xml | 4 ----
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 0abf9a0..945e8ce 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -78,6 +78,7 @@ public class MRJobParser implements Runnable {
private List<String> configKeys;
private static final int TOP_BOTTOM_TASKS_BY_ELAPSED_TIME = 10;
private static final int FLUSH_TASKS_EVERY_TIME = 5;
+ private static final int MAX_TASKS_PERMIT = 5000;
static {
OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
@@ -428,6 +429,16 @@ public class MRJobParser implements Runnable {
}
private Function<String, Boolean> fetchTasks = jobId -> {
+ try {
+ JobExecutionAPIEntity entity = this.mrJobEntityMap.get(jobId);
+ int taskNumber = entity.getNumTotalMaps() + entity.getNumTotalReduces();
+ if (taskNumber > MAX_TASKS_PERMIT) {
+ LOG.info("too many tasks {}, ignore tasks", taskNumber);
+ return true;
+ }
+ } catch (Exception e) {
+ return true;
+ }
String taskURL = app.getTrackingUrl() + Constants.MR_JOBS_URL + "/" + jobId + "/" + Constants.MR_TASKS_URL + "?" + Constants.ANONYMOUS_PARAMETER;
InputStream is = null;
List<MRTask> tasks = null;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/15f39705/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index a9dddb0..f5aee5f 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -42,10 +42,6 @@
<version>${project.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>