You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/08/24 07:35:30 UTC
incubator-eagle git commit: [EAGLE-494] add fields to mr history job
entity
Repository: incubator-eagle
Updated Branches:
refs/heads/develop 97c5f1ec9 -> b52405f12
[EAGLE-494] add fields to mr history job entity
Author: wujinhu <wu...@126.com>
Closes #377 from wujinhu/EAGLE-494.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b52405f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b52405f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b52405f1
Branch: refs/heads/develop
Commit: b52405f128dee0354c1791bd4b7995a730c9050c
Parents: 97c5f1e
Author: wujinhu <wu...@126.com>
Authored: Wed Aug 24 15:35:17 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Wed Aug 24 15:35:17 2016 +0800
----------------------------------------------------------------------
.../mr/historyentity/JobExecutionAPIEntity.java | 44 ++++++++++++++++++++
.../mr/history/parser/JHFEventReaderBase.java | 26 ++++++++++++
2 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b52405f1/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
index db8f0d6..d9093ff 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
@@ -73,6 +73,14 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity {
private long lastReduceDuration;
@Column("s")
private long durationTime;
+ @Column("u")
+ private double avgMapTaskDuration;
+ @Column("v")
+ private double avgReduceTaskDuration;
+ @Column("w")
+ private double maxMapTaskDuration;
+ @Column("x")
+ private double maxReduceTaskDuration;
public long getDurationTime() {
return durationTime;
@@ -227,4 +235,40 @@ public class JobExecutionAPIEntity extends JobBaseAPIEntity {
this.lastReduceDuration = lastReduceDuration;
valueChanged("lastReduceDuration");
}
+
+ public double getAvgMapTaskDuration() {
+ return avgMapTaskDuration;
+ }
+
+ public void setAvgMapTaskDuration(double avgMapTaskDuration) {
+ this.avgMapTaskDuration = avgMapTaskDuration;
+ valueChanged("avgMapTaskDuration");
+ }
+
+ public double getAvgReduceTaskDuration() {
+ return avgReduceTaskDuration;
+ }
+
+ public void setAvgReduceTaskDuration(double avgReduceTaskDuration) {
+ this.avgReduceTaskDuration = avgReduceTaskDuration;
+ valueChanged("avgReduceTaskDuration");
+ }
+
+ public double getMaxMapTaskDuration() {
+ return maxMapTaskDuration;
+ }
+
+ public void setMaxMapTaskDuration(double maxMapTaskDuration) {
+ this.maxMapTaskDuration = maxMapTaskDuration;
+ valueChanged("maxMapTaskDuration");
+ }
+
+ public double getMaxReduceTaskDuration() {
+ return maxReduceTaskDuration;
+ }
+
+ public void setMaxReduceTaskDuration(double maxReduceTaskDuration) {
+ this.maxReduceTaskDuration = maxReduceTaskDuration;
+ valueChanged("maxReduceTaskDuration");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b52405f1/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
index af3df85..9992690 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
@@ -71,6 +71,9 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl
protected final Configuration configuration;
+ private long sumMapTaskDuration;
+ private long sumReduceTaskDuration;
+
public Constants.JobType fetchJobType(Configuration config) {
if (config.get(Constants.JobConfiguration.CASCADING_JOB) != null) { return Constants.JobType.CASCADING; }
if (config.get(Constants.JobConfiguration.HIVE_JOB) != null) { return Constants.JobType.HIVE; }
@@ -113,6 +116,8 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl
if (this.configuration != null && this.m_jobType == null) {
this.setJobType(fetchJobType(this.configuration).toString());
}
+ this.sumMapTaskDuration = 0l;
+ this.sumReduceTaskDuration = 0l;
}
public void register(HistoryJobEntityLifecycleListener lifecycleListener){
@@ -261,6 +266,13 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl
m_jobExecutionEntity.setRackLocalMapsPercentage(m_jobExecutionEntity.getRackLocalMaps() * 1.0 / m_jobExecutionEntity.getTotalLaunchedMaps());
}
}
+ m_jobExecutionEntity.setAvgMapTaskDuration(this.sumMapTaskDuration * 1.0 / m_numTotalMaps);
+ if (m_numTotalReduces == 0) {
+ m_jobExecutionEntity.setMaxReduceTaskDuration(0);
+ m_jobExecutionEntity.setAvgReduceTaskDuration(0);
+ } else {
+ m_jobExecutionEntity.setAvgReduceTaskDuration(this.sumReduceTaskDuration * 1.0 / m_numTotalReduces);
+ }
entityCreated(m_jobExecutionEntity);
}
}
@@ -333,6 +345,20 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl
if (taskType.equals(Constants.TaskType.REDUCE.toString()) && duration > m_jobExecutionEntity.getLastReduceDuration()) {
m_jobExecutionEntity.setLastReduceDuration(duration);
}
+
+ if (taskType.equals(Constants.TaskType.MAP.toString()) && entity.getDuration() > m_jobExecutionEntity.getMaxMapTaskDuration()) {
+ m_jobExecutionEntity.setMaxMapTaskDuration(entity.getDuration());
+ }
+ if (taskType.equals(Constants.TaskType.REDUCE.toString()) && entity.getDuration() > m_jobExecutionEntity.getMaxReduceTaskDuration()) {
+ m_jobExecutionEntity.setMaxReduceTaskDuration(entity.getDuration());
+ }
+
+ if (taskType.equals(Constants.TaskType.MAP.toString())) {
+ this.sumMapTaskDuration += entity.getDuration();
+ } else {
+ this.sumReduceTaskDuration += entity.getDuration();
+ }
+
entityCreated(entity);
//_taskStartTime.remove(taskID); // clean this taskID
} else if ((recType == RecordTypes.MapAttempt || recType == RecordTypes.ReduceAttempt) && startTime != null) { // task attempt start