You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2016/07/08 17:52:42 UTC
[48/50] [abbrv] hadoop git commit: MAPREDUCE-6720. Inconsistent
values of counters across tasks and job reported to timeline service.
Contributed by Varun Saxena
MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c524bde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c524bde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c524bde
Branch: refs/heads/YARN-2928
Commit: 4c524bde62d55b6144e94285350d16685bb85351
Parents: 62da809
Author: Naganarasimha <na...@apache.org>
Authored: Wed Jun 22 23:32:19 2016 +0530
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Jul 8 10:19:09 2016 -0700
----------------------------------------------------------------------
.../mapreduce/jobhistory/JobHistoryEventHandler.java | 4 ++++
.../hadoop/mapreduce/jobhistory/JobFinishedEvent.java | 12 ++++++------
.../hadoop/mapreduce/util/JobHistoryEventUtils.java | 7 ++++++-
3 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c524bde/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a09c17b..9b59676 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1125,6 +1125,10 @@ public class JobHistoryEventHandler extends AbstractService
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskId);
+ if (event.getEventType() == EventType.TASK_STARTED) {
+ entity.addInfo("TASK_TYPE",
+ ((TaskStartedEvent)event).getTaskType().toString());
+ }
entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
return entity;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c524bde/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 76d6897..ea21f60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -158,12 +158,12 @@ public class JobFinishedEvent implements HistoryEvent {
@Override
public Set<TimelineMetric> getTimelineMetrics() {
- Set<TimelineMetric> jobMetrics = JobHistoryEventUtils
- .countersToTimelineMetric(getMapCounters(), finishTime);
- jobMetrics.addAll(JobHistoryEventUtils
- .countersToTimelineMetric(getReduceCounters(), finishTime));
- jobMetrics.addAll(JobHistoryEventUtils
- .countersToTimelineMetric(getTotalCounters(), finishTime));
+ Set<TimelineMetric> jobMetrics = JobHistoryEventUtils.
+ countersToTimelineMetric(getTotalCounters(), finishTime);
+ jobMetrics.addAll(JobHistoryEventUtils.
+ countersToTimelineMetric(getMapCounters(), finishTime, "MAP:"));
+ jobMetrics.addAll(JobHistoryEventUtils.
+ countersToTimelineMetric(getReduceCounters(), finishTime, "REDUCE:"));
return jobMetrics;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c524bde/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index 35d066c..89abf0e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -61,11 +61,16 @@ public final class JobHistoryEventUtils {
public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
long timestamp) {
+ return countersToTimelineMetric(counters, timestamp, "");
+ }
+
+ public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
+ long timestamp, String groupNamePrefix) {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
for (CounterGroup g : counters) {
String groupName = g.getName();
for (Counter c : g) {
- String name = groupName + ":" + c.getName();
+ String name = groupNamePrefix + groupName + ":" + c.getName();
TimelineMetric metric = new TimelineMetric();
metric.setId(name);
metric.addValue(timestamp, c.getValue());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org