You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/05/02 19:22:17 UTC
[42/50] [abbrv] hadoop git commit: MAPREDUCE-7072. mapred job
-history prints duplicate counter in human output (wilfreds via rkanter)
MAPREDUCE-7072. mapred job -history prints duplicate counter in human output (wilfreds via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b47275fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b47275fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b47275fe
Branch: refs/heads/YARN-8200
Commit: b47275fe28f51d14adf777822b45c0812e71a2a6
Parents: 600f4d4
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Apr 27 09:57:31 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Apr 27 09:57:31 2018 -0700
----------------------------------------------------------------------
.../HumanReadableHistoryViewerPrinter.java | 3 +-
.../jobhistory/JSONHistoryViewerPrinter.java | 3 +-
.../jobhistory/TestHistoryViewerPrinter.java | 76 ++++++++++++++++++++
3 files changed, 80 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
index d3da9f4..fdf3c47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HumanReadableHistoryViewerPrinter.java
@@ -148,7 +148,8 @@ class HumanReadableHistoryViewerPrinter implements HistoryViewerPrinter {
"Total Value"));
buff.append("\n------------------------------------------" +
"---------------------------------------------");
- for (String groupName : totalCounters.getGroupNames()) {
+ for (CounterGroup counterGroup : totalCounters) {
+ String groupName = counterGroup.getName();
CounterGroup totalGroup = totalCounters.getGroup(groupName);
CounterGroup mapGroup = mapCounters.getGroup(groupName);
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
index 456dcf7..850fe2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JSONHistoryViewerPrinter.java
@@ -104,7 +104,8 @@ class JSONHistoryViewerPrinter implements HistoryViewerPrinter {
// Killed jobs might not have counters
if (totalCounters != null) {
JSONObject jGroups = new JSONObject();
- for (String groupName : totalCounters.getGroupNames()) {
+ for (CounterGroup counterGroup : totalCounters) {
+ String groupName = counterGroup.getName();
CounterGroup totalGroup = totalCounters.getGroup(groupName);
CounterGroup mapGroup = mapCounters.getGroup(groupName);
CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47275fe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
index 358e9b2..74a2605 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
@@ -859,6 +859,46 @@ public class TestHistoryViewerPrinter {
"}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
}
+ @Test
+ public void testHumanDupePrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo2();
+ // Counters are only part of the overview so printAll can be false or true
+ // this does not affect the test, task counters are not printed
+ HumanReadableHistoryViewerPrinter printer =
+ new HumanReadableHistoryViewerPrinter(job, false, "http://",
+ TimeZone.getTimeZone("GMT"));
+ String outStr = run(printer);
+ // We are not interested in anything but the duplicate counter
+ int count1 = outStr.indexOf(
+ "|Map-Reduce Framework |Map input records |");
+ Assert.assertNotEquals("First counter occurrence not found", -1, count1);
+ int count2 = outStr.indexOf(
+ "|Map-Reduce Framework |Map input records |",
+ count1 + 1);
+ Assert.assertEquals("Duplicate counter found at: " + count1 +
+ " and " + count2, -1, count2);
+ }
+
+ @Test
+ public void testJSONDupePrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo2();
+ // Counters are part of the overview and task info
+ // Tasks only have bogus counters in the test if that is changed printAll
+ // must then be kept as false for this test to pass
+ JSONHistoryViewerPrinter printer =
+ new JSONHistoryViewerPrinter(job, false, "http://");
+ String outStr = run(printer);
+ // We are not interested in anything but the duplicate counter
+ int count1 = outStr.indexOf(
+ "\"counterName\":\"MAP_INPUT_RECORDS\"");
+ Assert.assertNotEquals("First counter occurrence not found", -1, count1);
+ int count2 = outStr.indexOf(
+ "\"counterName\":\"MAP_INPUT_RECORDS\"",
+ count1 + 1);
+ Assert.assertEquals("Duplicate counter found at: " + count1 +
+ " and " + count2, -1, count2);
+ }
+
private String run(HistoryViewerPrinter printer) throws Exception {
ByteArrayOutputStream boas = new ByteArrayOutputStream();
PrintStream out = new PrintStream(boas, true);
@@ -901,6 +941,32 @@ public class TestHistoryViewerPrinter {
addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
return job;
}
+ private static JobHistoryParser.JobInfo createJobInfo2() {
+ JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
+ job.submitTime = 1317928501754L;
+ job.finishTime = job.submitTime + 15000;
+ job.jobid = JobID.forName("job_1317928501754_0001");
+ job.username = "test";
+ job.jobname = "Dupe counter output";
+ job.jobQueueName = "root.test";
+ job.jobConfPath = "/tmp/job.xml";
+ job.launchTime = job.submitTime + 1000;
+ job.totalMaps = 1;
+ job.totalReduces = 0;
+ job.finishedMaps = 1;
+ job.finishedReduces = 0;
+ job.failedMaps = 0;
+ job.failedReduces = 0;
+ job.jobStatus = JobStatus.State.SUCCEEDED.name();
+ job.totalCounters = createDeprecatedCounters();
+ job.mapCounters = createDeprecatedCounters();
+ job.reduceCounters = createDeprecatedCounters();
+ job.tasksMap = new HashMap<>();
+ addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED);
+ return job;
+ }
private static Counters createCounters() {
Counters counters = new Counters();
@@ -910,6 +976,16 @@ public class TestHistoryViewerPrinter {
return counters;
}
+ private static Counters createDeprecatedCounters() {
+ Counters counters = new Counters();
+ // Deprecated counter: make sure it is only printed once
+ counters.findCounter("org.apache.hadoop.mapred.Task$Counter",
+ "MAP_INPUT_RECORDS").setValue(1);
+ counters.findCounter("File System Counters",
+ "FILE: Number of bytes read").setValue(1);
+ return counters;
+ }
+
private static void addTaskInfo(JobHistoryParser.JobInfo job,
TaskType type, int id, TaskStatus.State status) {
JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org