You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by na...@apache.org on 2016/06/22 18:05:50 UTC
hadoop git commit: MAPREDUCE-6720. Inconsistent values of counters
across tasks and job reported to timeline service. Contributed by Varun
Saxena
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 92faa0152 -> 52a4cfe53
MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52a4cfe5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52a4cfe5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52a4cfe5
Branch: refs/heads/YARN-2928
Commit: 52a4cfe53339569c08d445628fab8a1d8ce52271
Parents: 92faa01
Author: Naganarasimha <na...@apache.org>
Authored: Wed Jun 22 23:32:19 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Jun 22 23:32:19 2016 +0530
----------------------------------------------------------------------
.../mapreduce/jobhistory/JobHistoryEventHandler.java | 4 ++++
.../hadoop/mapreduce/jobhistory/JobFinishedEvent.java | 12 ++++++------
.../hadoop/mapreduce/util/JobHistoryEventUtils.java | 7 ++++++-
3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a4cfe5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a09c17b..9b59676 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1125,6 +1125,10 @@ public class JobHistoryEventHandler extends AbstractService
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskId);
+ if (event.getEventType() == EventType.TASK_STARTED) {
+ entity.addInfo("TASK_TYPE",
+ ((TaskStartedEvent)event).getTaskType().toString());
+ }
entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
return entity;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a4cfe5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 76d6897..ea21f60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -158,12 +158,12 @@ public class JobFinishedEvent implements HistoryEvent {
@Override
public Set<TimelineMetric> getTimelineMetrics() {
- Set<TimelineMetric> jobMetrics = JobHistoryEventUtils
- .countersToTimelineMetric(getMapCounters(), finishTime);
- jobMetrics.addAll(JobHistoryEventUtils
- .countersToTimelineMetric(getReduceCounters(), finishTime));
- jobMetrics.addAll(JobHistoryEventUtils
- .countersToTimelineMetric(getTotalCounters(), finishTime));
+ Set<TimelineMetric> jobMetrics = JobHistoryEventUtils.
+ countersToTimelineMetric(getTotalCounters(), finishTime);
+ jobMetrics.addAll(JobHistoryEventUtils.
+ countersToTimelineMetric(getMapCounters(), finishTime, "MAP:"));
+ jobMetrics.addAll(JobHistoryEventUtils.
+ countersToTimelineMetric(getReduceCounters(), finishTime, "REDUCE:"));
return jobMetrics;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a4cfe5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index 35d066c..89abf0e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -61,11 +61,16 @@ public final class JobHistoryEventUtils {
public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
long timestamp) {
+ return countersToTimelineMetric(counters, timestamp, "");
+ }
+
+ public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
+ long timestamp, String groupNamePrefix) {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
for (CounterGroup g : counters) {
String groupName = g.getName();
for (Counter c : g) {
- String name = groupName + ":" + c.getName();
+ String name = groupNamePrefix + groupName + ":" + c.getName();
TimelineMetric metric = new TimelineMetric();
metric.setId(name);
metric.addValue(timestamp, c.getValue());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org