You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2015/07/31 20:20:20 UTC
hadoop git commit: MAPREDUCE-6394. Speed up Task processing loop in
HsTasksBlock#render(). Contributed by Ray Chiang (cherry picked from commit
d0e0ba8010b72f58ddede5303f2b88404263d4bf)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c31e3ba92 -> 508da4d46
MAPREDUCE-6394. Speed up Task processing loop in HsTasksBlock#render(). Contributed by Ray Chiang
(cherry picked from commit d0e0ba8010b72f58ddede5303f2b88404263d4bf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/508da4d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/508da4d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/508da4d4
Branch: refs/heads/branch-2
Commit: 508da4d461a3bdf3239e627b50a37faa3b8d9422
Parents: c31e3ba
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jul 31 18:17:54 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jul 31 18:19:52 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../v2/api/records/TestTaskAttemptReport.java | 131 +++++++++++++++++
.../v2/api/records/TestTaskReport.java | 139 +++++++++++++++++++
.../v2/api/records/TaskAttemptReport.java | 3 +
.../mapreduce/v2/api/records/TaskReport.java | 5 +-
.../impl/pb/TaskAttemptReportPBImpl.java | 38 ++++-
.../api/records/impl/pb/TaskReportPBImpl.java | 38 +++--
.../hadoop/mapreduce/v2/hs/CompletedTask.java | 2 +-
.../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +-
9 files changed, 343 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 704b170..5eff741 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -108,6 +108,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
jlowe)
+ MAPREDUCE-6394. Speed up Task processing loop in HsTasksBlock#render()
+ (Ray Chiang via jlowe)
+
BUG FIXES
MAPREDUCE-6314. TestPipeApplication fails on trunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskAttemptReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskAttemptReport.java
new file mode 100644
index 0000000..cd7f758
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskAttemptReport.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.api.records;
+
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.records.impl.pb.TaskAttemptReportPBImpl;
+import org.apache.hadoop.mapreduce.v2.app.MockJobs;
+import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
+import org.apache.hadoop.yarn.util.Records;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestTaskAttemptReport {
+
+ @Test
+ public void testSetRawCounters() {
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ }
+
+ @Test
+ public void testBuildImplicitRawCounters() {
+ TaskAttemptReportPBImpl report = new TaskAttemptReportPBImpl();
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ MRProtos.TaskAttemptReportProto protoVal = report.getProto();
+ Counters counters = report.getCounters();
+ assertTrue(protoVal.hasCounters());
+ }
+
+ @Test
+ public void testCountersOverRawCounters() {
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ Counters altCounters = TypeConverter.toYarn(rCounters);
+ report.setRawCounters(rCounters);
+ report.setCounters(altCounters);
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ assertNotEquals(rCounters, altCounters);
+ assertEquals(counters, altCounters);
+ }
+
+ @Test
+ public void testUninitializedCounters() {
+ // Create basic class
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ // Verify properties initialized to null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetRawCountersToNull() {
+ // Create basic class
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ // Set raw counters to null
+ report.setRawCounters(null);
+ // Verify properties still null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+
+ }
+
+ @Test
+ public void testSetCountersToNull() {
+ // Create basic class
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ // Set raw counters to null
+ report.setCounters(null);
+ // Verify properties still null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetNonNullCountersToNull() {
+ // Create basic class
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ // Set raw counters
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ // Verify getCounters converts properly from raw to real
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ // Clear counters to null and then verify
+ report.setCounters(null);
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetNonNullRawCountersToNull() {
+ // Create basic class
+ TaskAttemptReport report = Records.newRecord(TaskAttemptReport.class);
+ // Set raw counters
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ // Verify getCounters converts properly from raw to real
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ // Clear counters to null and then verify
+ report.setRawCounters(null);
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskReport.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskReport.java
new file mode 100644
index 0000000..6801661
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/api/records/TestTaskReport.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.api.records;
+
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.api.records.impl.pb.TaskReportPBImpl;
+import org.apache.hadoop.mapreduce.v2.app.MockJobs;
+import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
+import org.apache.hadoop.yarn.util.Records;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestTaskReport {
+
+ @Test
+ public void testSetRawCounters() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ // Set raw counters
+ report.setRawCounters(rCounters);
+ // Verify getCounters converts properly from raw to real
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ }
+
+ @Test
+ public void testBuildImplicitRawCounters() {
+ // Create basic class
+ TaskReportPBImpl report = new TaskReportPBImpl();
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ // Set raw counters
+ report.setRawCounters(rCounters);
+ // Verify getProto method implicitly converts/sets real counters
+ MRProtos.TaskReportProto protoVal = report.getProto();
+ assertTrue(protoVal.hasCounters());
+ }
+
+ @Test
+ public void testCountersOverRawCounters() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ Counters altCounters = TypeConverter.toYarn(rCounters);
+ // Set raw counters
+ report.setRawCounters(rCounters);
+ // Set real counters
+ report.setCounters(altCounters);
+ // Verify real counters has priority over raw
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ assertNotEquals(rCounters, altCounters);
+ assertEquals(counters, altCounters);
+ }
+
+ @Test
+ public void testUninitializedCounters() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ // Verify properties initialized to null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetRawCountersToNull() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ // Set raw counters to null
+ report.setRawCounters(null);
+ // Verify properties still null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+
+ }
+
+ @Test
+ public void testSetCountersToNull() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ // Set raw counters to null
+ report.setCounters(null);
+ // Verify properties still null
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetNonNullCountersToNull() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ // Set raw counters
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ // Verify getCounters converts properly from raw to real
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ // Clear counters to null and then verify
+ report.setCounters(null);
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+
+ @Test
+ public void testSetNonNullRawCountersToNull() {
+ // Create basic class
+ TaskReport report = Records.newRecord(TaskReport.class);
+ // Set raw counters
+ org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters();
+ report.setRawCounters(rCounters);
+ // Verify getCounters converts properly from raw to real
+ Counters counters = report.getCounters();
+ assertNotEquals(null, counters);
+ // Clear counters to null and then verify
+ report.setRawCounters(null);
+ assertEquals(null, report.getCounters());
+ assertEquals(null, report.getRawCounters());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptReport.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptReport.java
index bc0a4c6..810887b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptReport.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptReport.java
@@ -31,6 +31,7 @@ public interface TaskAttemptReport {
/** @return the sort/merge finish time. Applicable only for reduce attempts */
public abstract long getSortFinishTime();
public abstract Counters getCounters();
+ public abstract org.apache.hadoop.mapreduce.Counters getRawCounters();
public abstract String getDiagnosticInfo();
public abstract String getStateString();
public abstract Phase getPhase();
@@ -45,6 +46,8 @@ public interface TaskAttemptReport {
public abstract void setStartTime(long startTime);
public abstract void setFinishTime(long finishTime);
public abstract void setCounters(Counters counters);
+ public abstract void
+ setRawCounters(org.apache.hadoop.mapreduce.Counters rCounters);
public abstract void setDiagnosticInfo(String diagnosticInfo);
public abstract void setStateString(String stateString);
public abstract void setPhase(Phase phase);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
index 1444a53..b7300c6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskReport.java
@@ -28,6 +28,7 @@ public interface TaskReport {
public abstract long getStartTime();
public abstract long getFinishTime();
public abstract Counters getCounters();
+ public abstract org.apache.hadoop.mapreduce.Counters getRawCounters();
public abstract List<TaskAttemptId> getRunningAttemptsList();
public abstract TaskAttemptId getRunningAttempt(int index);
public abstract int getRunningAttemptsCount();
@@ -46,7 +47,9 @@ public interface TaskReport {
public abstract void setStartTime(long startTime);
public abstract void setFinishTime(long finishTime);
public abstract void setCounters(Counters counters);
-
+ public abstract void
+ setRawCounters(org.apache.hadoop.mapreduce.Counters rCounters);
+
public abstract void addAllRunningAttempts(List<TaskAttemptId> taskAttempts);
public abstract void addRunningAttempt(TaskAttemptId taskAttempt);
public abstract void removeRunningAttempt(int index);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskAttemptReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskAttemptReportPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskAttemptReportPBImpl.java
index 96be84a..4677512 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskAttemptReportPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskAttemptReportPBImpl.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.mapreduce.v2.api.records.impl.pb;
-
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.Phase;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -42,12 +42,12 @@ public class TaskAttemptReportPBImpl extends ProtoBase<TaskAttemptReportProto> i
TaskAttemptReportProto proto = TaskAttemptReportProto.getDefaultInstance();
TaskAttemptReportProto.Builder builder = null;
boolean viaProto = false;
-
+
private TaskAttemptId taskAttemptId = null;
private Counters counters = null;
+ private org.apache.hadoop.mapreduce.Counters rawCounters = null;
private ContainerId containerId = null;
-
-
+
public TaskAttemptReportPBImpl() {
builder = TaskAttemptReportProto.newBuilder();
}
@@ -68,6 +68,7 @@ public class TaskAttemptReportPBImpl extends ProtoBase<TaskAttemptReportProto> i
if (this.taskAttemptId != null) {
builder.setTaskAttemptId(convertToProtoFormat(this.taskAttemptId));
}
+ convertRawCountersToCounters();
if (this.counters != null) {
builder.setCounters(convertToProtoFormat(this.counters));
}
@@ -90,11 +91,12 @@ public class TaskAttemptReportPBImpl extends ProtoBase<TaskAttemptReportProto> i
}
viaProto = false;
}
-
-
+
+
@Override
public Counters getCounters() {
TaskAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
+ convertRawCountersToCounters();
if (this.counters != null) {
return this.counters;
}
@@ -108,10 +110,32 @@ public class TaskAttemptReportPBImpl extends ProtoBase<TaskAttemptReportProto> i
@Override
public void setCounters(Counters counters) {
maybeInitBuilder();
- if (counters == null)
+ if (counters == null) {
builder.clearCounters();
+ }
this.counters = counters;
+ this.rawCounters = null;
}
+
+ @Override
+ public org.apache.hadoop.mapreduce.Counters
+ getRawCounters() {
+ return this.rawCounters;
+ }
+
+ @Override
+ public void setRawCounters(org.apache.hadoop.mapreduce.Counters rCounters) {
+ setCounters(null);
+ this.rawCounters = rCounters;
+ }
+
+ private void convertRawCountersToCounters() {
+ if (this.counters == null && this.rawCounters != null) {
+ this.counters = TypeConverter.toYarn(rawCounters);
+ this.rawCounters = null;
+ }
+ }
+
@Override
public long getStartTime() {
TaskAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
index ba1245c..638bb66 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/TaskReportPBImpl.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -37,21 +38,19 @@ import org.apache.hadoop.mapreduce.v2.proto.MRProtos.TaskStateProto;
import org.apache.hadoop.mapreduce.v2.util.MRProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
-
-
public class TaskReportPBImpl extends ProtoBase<TaskReportProto> implements TaskReport {
TaskReportProto proto = TaskReportProto.getDefaultInstance();
TaskReportProto.Builder builder = null;
boolean viaProto = false;
-
+
private TaskId taskId = null;
private Counters counters = null;
+ private org.apache.hadoop.mapreduce.Counters rawCounters = null;
private List<TaskAttemptId> runningAttempts = null;
private TaskAttemptId successfulAttemptId = null;
private List<String> diagnostics = null;
private String status;
-
-
+
public TaskReportPBImpl() {
builder = TaskReportProto.newBuilder();
}
@@ -72,6 +71,7 @@ public class TaskReportPBImpl extends ProtoBase<TaskReportProto> implements Task
if (this.taskId != null) {
builder.setTaskId(convertToProtoFormat(this.taskId));
}
+ convertRawCountersToCounters();
if (this.counters != null) {
builder.setCounters(convertToProtoFormat(this.counters));
}
@@ -100,11 +100,11 @@ public class TaskReportPBImpl extends ProtoBase<TaskReportProto> implements Task
}
viaProto = false;
}
-
-
+
@Override
public Counters getCounters() {
TaskReportProtoOrBuilder p = viaProto ? proto : builder;
+ convertRawCountersToCounters();
if (this.counters != null) {
return this.counters;
}
@@ -118,10 +118,32 @@ public class TaskReportPBImpl extends ProtoBase<TaskReportProto> implements Task
@Override
public void setCounters(Counters counters) {
maybeInitBuilder();
- if (counters == null)
+ if (counters == null) {
builder.clearCounters();
+ }
this.counters = counters;
+ this.rawCounters = null;
}
+
+ @Override
+ public org.apache.hadoop.mapreduce.Counters
+ getRawCounters() {
+ return this.rawCounters;
+ }
+
+ @Override
+ public void setRawCounters(org.apache.hadoop.mapreduce.Counters rCounters) {
+ setCounters(null);
+ this.rawCounters = rCounters;
+ }
+
+ private void convertRawCountersToCounters() {
+ if (this.counters == null && this.rawCounters != null) {
+ this.counters = TypeConverter.toYarn(rawCounters);
+ this.rawCounters = null;
+ }
+ }
+
@Override
public long getStartTime() {
TaskReportProtoOrBuilder p = viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
index 8469b27..81fddaf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
@@ -135,7 +135,7 @@ public class CompletedTask implements Task {
if (counters == null) {
counters = EMPTY_COUNTERS;
}
- report.setCounters(TypeConverter.toYarn(counters));
+ report.setRawCounters(counters);
if (successfulAttempt != null) {
report.setSuccessfulAttempt(successfulAttempt);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/508da4d4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
index 0aa2e0b..c87d82b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
@@ -170,7 +170,7 @@ public class CompletedTaskAttempt implements TaskAttempt {
}
// report.setPhase(attemptInfo.get); //TODO
report.setStateString(attemptInfo.getState());
- report.setCounters(TypeConverter.toYarn(getCounters()));
+ report.setRawCounters(getCounters());
report.setContainerId(attemptInfo.getContainerId());
if (attemptInfo.getHostname() == null) {
report.setNodeManagerHost("UNKNOWN");