You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/02/24 21:02:58 UTC
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6627. Add
machine-readable output to mapred job -history command (rkanter)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
new file mode 100644
index 0000000..358e9b2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestHistoryViewerPrinter.java
@@ -0,0 +1,945 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.jobhistory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.TaskStatus;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.junit.Assert;
+import org.junit.Test;
+import org.skyscreamer.jsonassert.JSONAssert;
+import org.skyscreamer.jsonassert.JSONCompareMode;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.TimeZone;
+
+public class TestHistoryViewerPrinter {
+
+ private static final Log LOG = LogFactory.getLog(
+ TestHistoryViewerPrinter.class);
+
+ @Test
+ public void testHumanPrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ HumanReadableHistoryViewerPrinter printer =
+ new HumanReadableHistoryViewerPrinter(job, false, "http://",
+ TimeZone.getTimeZone("GMT"));
+ String outStr = run(printer);
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ }
+
+ @Test
+ public void testHumanPrinterAll() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ HumanReadableHistoryViewerPrinter printer =
+ new HumanReadableHistoryViewerPrinter(job, true, "http://",
+ TimeZone.getTimeZone("GMT"));
+ String outStr = run(printer);
+ if (System.getProperty("java.version").startsWith("1.7")) {
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" +
+ "\n" +
+ "JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n" +
+ "\n" +
+ "MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n" +
+ "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n" +
+ "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n" +
+ "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n" +
+ "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n" +
+ "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n" +
+ "\n" +
+ "REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n" +
+ "\n" +
+ "JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ } else {
+ Assert.assertEquals("\n" +
+ "Hadoop job: job_1317928501754_0001\n" +
+ "=====================================\n" +
+ "User: rkanter\n" +
+ "JobName: my job\n" +
+ "JobConf: /tmp/job.xml\n" +
+ "Submitted At: 6-Oct-2011 19:15:01\n" +
+ "Launched At: 6-Oct-2011 19:15:02 (1sec)\n" +
+ "Finished At: 6-Oct-2011 19:15:16 (14sec)\n" +
+ "Status: SUCCEEDED\n" +
+ "Counters: \n" +
+ "\n" +
+ "|Group Name |Counter name |Map Value |Reduce Value|Total Value|\n" +
+ "---------------------------------------------------------------------------------------\n" +
+ "|group1 |counter1 |5 |5 |5 \n" +
+ "|group1 |counter2 |10 |10 |10 \n" +
+ "|group2 |counter1 |15 |15 |15 \n" +
+ "\n" +
+ "=====================================\n" +
+ "\n" +
+ "Task Summary\n" +
+ "============================\n" +
+ "Kind\tTotal\tSuccessful\tFailed\tKilled\tStartTime\tFinishTime\n" +
+ "\n" +
+ "Setup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\n" +
+ "Map\t6\t5\t\t1\t0\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:16 (12sec)\n" +
+ "Reduce\t1\t1\t\t0\t0\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\n" +
+ "Cleanup\t1\t1\t\t0\t0\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\n" +
+ "============================\n" +
+ "\n" +
+ "\n" +
+ "Analysis\n" +
+ "=========\n" +
+ "\n" +
+ "Time taken by best performing map task task_1317928501754_0001_m_000003: 3sec\n" +
+ "Average time taken by map tasks: 5sec\n" +
+ "Worse performing map tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_m_000007 7sec\n" +
+ "task_1317928501754_0001_m_000006 6sec\n" +
+ "task_1317928501754_0001_m_000005 5sec\n" +
+ "task_1317928501754_0001_m_000004 4sec\n" +
+ "task_1317928501754_0001_m_000003 3sec\n" +
+ "The last map task task_1317928501754_0001_m_000007 finished at (relative to the Job launch time): 6-Oct-2011 19:15:16 (14sec)\n" +
+ "\n" +
+ "Time taken by best performing shuffle task task_1317928501754_0001_r_000008: 8sec\n" +
+ "Average time taken by shuffle tasks: 8sec\n" +
+ "Worse performing shuffle tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 8sec\n" +
+ "The last shuffle task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "\n" +
+ "Time taken by best performing reduce task task_1317928501754_0001_r_000008: 0sec\n" +
+ "Average time taken by reduce tasks: 0sec\n" +
+ "Worse performing reduce tasks: \n" +
+ "TaskId\t\tTimetaken\n" +
+ "task_1317928501754_0001_r_000008 0sec\n" +
+ "The last reduce task task_1317928501754_0001_r_000008 finished at (relative to the Job launch time): 6-Oct-2011 19:15:18 (16sec)\n" +
+ "=========\n" +
+ "\n" +
+ "FAILED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000002\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_s_000001\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000007\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000006\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000005\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000004\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\tInputSplits\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_m_000003\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\t\t\n" +
+ "\n" +
+ "SUCCEEDED REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_r_000008\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t\n" +
+ "\n" +
+ "SUCCEEDED JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tError\n" +
+ "====================================================\n" +
+ "task_1317928501754_0001_c_000009\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\t\n" +
+ "\n" +
+ "JOB_SETUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_s_000001_1\t6-Oct-2011 19:15:03\t6-Oct-2011 19:15:04 (1sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\n" +
+ "\n" +
+ "MAP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_m_000007_1\t6-Oct-2011 19:15:09\t6-Oct-2011 19:15:16 (7sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\n" +
+ "attempt_1317928501754_0001_m_000002_1\t6-Oct-2011 19:15:04\t6-Oct-2011 19:15:06 (2sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\n" +
+ "attempt_1317928501754_0001_m_000006_1\t6-Oct-2011 19:15:08\t6-Oct-2011 19:15:14 (6sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\n" +
+ "attempt_1317928501754_0001_m_000005_1\t6-Oct-2011 19:15:07\t6-Oct-2011 19:15:12 (5sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\n" +
+ "attempt_1317928501754_0001_m_000004_1\t6-Oct-2011 19:15:06\t6-Oct-2011 19:15:10 (4sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\n" +
+ "attempt_1317928501754_0001_m_000003_1\t6-Oct-2011 19:15:05\t6-Oct-2011 19:15:08 (3sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\n" +
+ "\n" +
+ "REDUCE task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tShuffleFinished\tSortFinished\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_r_000008_1\t6-Oct-2011 19:15:10\t6-Oct-2011 19:15:18 (8sec)\t6-Oct-2011 19:15:18 (0sec)6-Oct-2011 19:15:18 (8sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\n" +
+ "\n" +
+ "JOB_CLEANUP task list for job_1317928501754_0001\n" +
+ "TaskId\t\tStartTime\tFinishTime\tHostName\tError\tTaskLogs\n" +
+ "====================================================\n" +
+ "attempt_1317928501754_0001_c_000009_1\t6-Oct-2011 19:15:11\t6-Oct-2011 19:15:20 (9sec)\tlocalhost\thttp://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\n" +
+ "\n" +
+ "FAILED task attempts by nodes\n" +
+ "Hostname\tFailedTasks\n" +
+ "===============================\n" +
+ "localhost\ttask_1317928501754_0001_m_000002, \n", outStr);
+ }
+ }
+
+ @Test
+ public void testJSONPrinter() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ JSONHistoryViewerPrinter printer =
+ new JSONHistoryViewerPrinter(job, false, "http://");
+ String outStr = run(printer);
+ JSONAssert.assertEquals("{\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 5,\n" +
+ " \"reduceValue\": 5,\n" +
+ " \"totalValue\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"mapValue\": 10,\n" +
+ " \"reduceValue\": 10,\n" +
+ " \"totalValue\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 15,\n" +
+ " \"reduceValue\": 15,\n" +
+ " \"totalValue\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishedAt\": 1317928516754,\n" +
+ " \"hadoopJob\": \"job_1317928501754_0001\",\n" +
+ " \"jobConf\": \"/tmp/job.xml\",\n" +
+ " \"jobName\": \"my job\",\n" +
+ " \"launchedAt\": 1317928502754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"submittedAt\": 1317928501754,\n" +
+ " \"taskSummary\": {\n" +
+ " \"cleanup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"map\": {\n" +
+ " \"failed\": 1,\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"successful\": 5,\n" +
+ " \"total\": 6\n" +
+ " },\n" +
+ " \"reduce\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"setup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " }\n" +
+ " },\n" +
+ " \"tasks\": [\n" +
+ " {\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"status\": \"FAILED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"user\": \"rkanter\"\n" +
+ "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
+ }
+
+ @Test
+ public void testJSONPrinterAll() throws Exception {
+ JobHistoryParser.JobInfo job = createJobInfo();
+ JSONHistoryViewerPrinter printer =
+ new JSONHistoryViewerPrinter(job, true, "http://");
+ String outStr = run(printer);
+ JSONAssert.assertEquals("{\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 5,\n" +
+ " \"reduceValue\": 5,\n" +
+ " \"totalValue\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"mapValue\": 10,\n" +
+ " \"reduceValue\": 10,\n" +
+ " \"totalValue\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"mapValue\": 15,\n" +
+ " \"reduceValue\": 15,\n" +
+ " \"totalValue\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishedAt\": 1317928516754,\n" +
+ " \"hadoopJob\": \"job_1317928501754_0001\",\n" +
+ " \"jobConf\": \"/tmp/job.xml\",\n" +
+ " \"jobName\": \"my job\",\n" +
+ " \"launchedAt\": 1317928502754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"submittedAt\": 1317928501754,\n" +
+ " \"taskSummary\": {\n" +
+ " \"cleanup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"map\": {\n" +
+ " \"failed\": 1,\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"successful\": 5,\n" +
+ " \"total\": 6\n" +
+ " },\n" +
+ " \"reduce\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " },\n" +
+ " \"setup\": {\n" +
+ " \"failed\": 0,\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"killed\": 0,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"successful\": 1,\n" +
+ " \"total\": 1\n" +
+ " }\n" +
+ " },\n" +
+ " \"tasks\": [\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000002_1\",\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000002_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928506754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928504754,\n" +
+ " \"status\": \"FAILED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000002\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_s_000001_1\",\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_s_000001_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928504754,\n" +
+ " \"startTime\": 1317928503754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_s_000001\",\n" +
+ " \"type\": \"JOB_SETUP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000006_1\",\n" +
+ " \"finishTime\": 1317928514754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928508754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000006_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928514754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928508754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000006\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000005_1\",\n" +
+ " \"finishTime\": 1317928512754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928507754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000005_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928512754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928507754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000005\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000004_1\",\n" +
+ " \"finishTime\": 1317928510754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928506754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000004_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928510754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928506754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000004\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000003_1\",\n" +
+ " \"finishTime\": 1317928508754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928505754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000003_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928508754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928505754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000003\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_c_000009_1\",\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_c_000009_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928520754,\n" +
+ " \"startTime\": 1317928511754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_c_000009\",\n" +
+ " \"type\": \"JOB_CLEANUP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_m_000007_1\",\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"startTime\": 1317928509754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_m_000007_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928516754,\n" +
+ " \"inputSplits\": \"\",\n" +
+ " \"startTime\": 1317928509754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_m_000007\",\n" +
+ " \"type\": \"MAP\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"attempts\": {\n" +
+ " \"attemptId\": \"attempt_1317928501754_0001_r_000008_1\",\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"hostName\": \"localhost\",\n" +
+ " \"shuffleFinished\": 1317928518754,\n" +
+ " \"sortFinished\": 1317928518754,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"taskLogs\": \"http://t:1234/tasklog?attemptid=attempt_1317928501754_0001_r_000008_1\"\n" +
+ " },\n" +
+ " \"counters\": {\n" +
+ " \"group1\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 5\n" +
+ " },\n" +
+ " {\n" +
+ " \"counterName\": \"counter2\",\n" +
+ " \"value\": 10\n" +
+ " }\n" +
+ " ],\n" +
+ " \"group2\": [\n" +
+ " {\n" +
+ " \"counterName\": \"counter1\",\n" +
+ " \"value\": 15\n" +
+ " }\n" +
+ " ]\n" +
+ " },\n" +
+ " \"finishTime\": 1317928518754,\n" +
+ " \"startTime\": 1317928510754,\n" +
+ " \"status\": \"SUCCEEDED\",\n" +
+ " \"taskId\": \"task_1317928501754_0001_r_000008\",\n" +
+ " \"type\": \"REDUCE\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"user\": \"rkanter\"\n" +
+ "}\n", outStr, JSONCompareMode.NON_EXTENSIBLE);
+ }
+
+ private String run(HistoryViewerPrinter printer) throws Exception {
+ ByteArrayOutputStream boas = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(boas, true);
+ printer.print(out);
+ out.close();
+ String outStr = boas.toString("UTF-8");
+ LOG.info("out = " + outStr);
+ return outStr;
+ }
+
+ private static JobHistoryParser.JobInfo createJobInfo() {
+ JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo();
+ job.submitTime = 1317928501754L;
+ job.finishTime = job.submitTime + 15000;
+ job.jobid = JobID.forName("job_1317928501754_0001");
+ job.username = "rkanter";
+ job.jobname = "my job";
+ job.jobQueueName = "my queue";
+ job.jobConfPath = "/tmp/job.xml";
+ job.launchTime = job.submitTime + 1000;
+ job.totalMaps = 5;
+ job.totalReduces = 1;
+ job.failedMaps = 1;
+ job.failedReduces = 0;
+ job.finishedMaps = 5;
+ job.finishedReduces = 1;
+ job.jobStatus = JobStatus.State.SUCCEEDED.name();
+ job.totalCounters = createCounters();
+ job.mapCounters = createCounters();
+ job.reduceCounters = createCounters();
+ job.tasksMap = new HashMap<>();
+ addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.FAILED);
+ addTaskInfo(job, TaskType.MAP, 3, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 4, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 5, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 6, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.MAP, 7, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.REDUCE, 8, TaskStatus.State.SUCCEEDED);
+ addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED);
+ return job;
+ }
+
+ private static Counters createCounters() {
+ Counters counters = new Counters();
+ counters.findCounter("group1", "counter1").setValue(5);
+ counters.findCounter("group1", "counter2").setValue(10);
+ counters.findCounter("group2", "counter1").setValue(15);
+ return counters;
+ }
+
+ private static void addTaskInfo(JobHistoryParser.JobInfo job,
+ TaskType type, int id, TaskStatus.State status) {
+ JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo();
+ task.taskId = new TaskID(job.getJobId(), type, id);
+ task.startTime = job.getLaunchTime() + id * 1000;
+ task.finishTime = task.startTime + id * 1000;
+ task.taskType = type;
+ task.counters = createCounters();
+ task.status = status.name();
+ task.attemptsMap = new HashMap<>();
+ addTaskAttemptInfo(task, 1);
+ job.tasksMap.put(task.getTaskId(), task);
+ }
+
+ private static void addTaskAttemptInfo(
+ JobHistoryParser.TaskInfo task, int id) {
+ JobHistoryParser.TaskAttemptInfo attempt =
+ new JobHistoryParser.TaskAttemptInfo();
+ attempt.attemptId = new TaskAttemptID(
+ TaskID.downgrade(task.getTaskId()), id);
+ attempt.startTime = task.getStartTime();
+ attempt.finishTime = task.getFinishTime();
+ attempt.shuffleFinishTime = task.getFinishTime();
+ attempt.sortFinishTime = task.getFinishTime();
+ attempt.mapFinishTime = task.getFinishTime();
+ attempt.status = task.getTaskStatus();
+ attempt.taskType = task.getTaskType();
+ attempt.trackerName = "localhost";
+ attempt.httpPort = 1234;
+ attempt.hostname = "localhost";
+ task.attemptsMap.put(attempt.getAttemptId(), attempt);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 2382239..f13c163 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
@@ -29,6 +30,8 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -359,27 +362,152 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
String historyFileUri = new Path(f.getAbsolutePath())
.makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toUri()
.toString();
-
- // bad command
- int exitCode = runTool(conf, jc, new String[] { "-history", "pul",
- historyFileUri }, out);
- assertEquals("Exit code", -1, exitCode);
- exitCode = runTool(conf, jc, new String[] { "-history", "all",
- historyFileUri }, out);
+ // Try a bunch of different valid combinations of the command to test
+ // argument parsing
+ int exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ }, out);
assertEquals("Exit code", 0, exitCode);
- String line;
+ checkHistoryHumanOutput(out);
+ File outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath()
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanFileOutput(out, outFile);
+ outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath(),
+ "-format",
+ "human"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanFileOutput(out, outFile);
+
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format",
+ "human"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryHumanOutput(out);
+
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONOutput(out);
+ outFile = File.createTempFile("myout", ".txt");
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ "all",
+ historyFileUri,
+ "-outfile",
+ outFile.getAbsolutePath(),
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONFileOutput(out, outFile);
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format",
+ "json"
+ }, out);
+ assertEquals("Exit code", 0, exitCode);
+ checkHistoryJSONOutput(out);
+
+ // Check some bad arguments
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "foo"
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ exitCode = runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-format"
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ runTool(conf, jc, new String[] {
+ "-history",
+ historyFileUri,
+ "-outfile",
+ }, out);
+ assertEquals("Exit code", -1, exitCode);
+ try {
+ runTool(conf, jc, new String[]{
+ "-history",
+ historyFileUri,
+ "-format",
+ "foo"
+ }, out);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+ }
+
+ private void checkHistoryHumanOutput(ByteArrayOutputStream out)
+ throws IOException, JSONException {
BufferedReader br = new BufferedReader(new InputStreamReader(
new ByteArrayInputStream(out.toByteArray())));
- int counter = 0;
- while ((line = br.readLine()) != null) {
- LOG.info("line = " + line);
- if (line.startsWith("task_")) {
- counter++;
- }
- }
- assertEquals(23, counter);
+ br.readLine();
+ String line = br.readLine();
+ br.close();
+ assertEquals("Hadoop job: job_1329348432655_0001", line);
+ out.reset();
+ }
+
+ private void checkHistoryJSONOutput(ByteArrayOutputStream out)
+ throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ new ByteArrayInputStream(out.toByteArray())));
+ String line = org.apache.commons.io.IOUtils.toString(br);
+ br.close();
+ JSONObject json = new JSONObject(line);
+ assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+ out.reset();
+ }
+
+ private void checkHistoryHumanFileOutput(ByteArrayOutputStream out,
+ File outFile) throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ br.readLine();
+ String line = br.readLine();
+ br.close();
+ assertEquals("Hadoop job: job_1329348432655_0001", line);
+ assertEquals(0, out.size());
+ }
+
+ private void checkHistoryJSONFileOutput(ByteArrayOutputStream out,
+ File outFile) throws IOException, JSONException {
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ String line = org.apache.commons.io.IOUtils.toString(br);
+ br.close();
+ JSONObject json = new JSONObject(line);
+ assertEquals("job_1329348432655_0001", json.getString("hadoopJob"));
+ assertEquals(0, out.size());
}
+
/**
* print job events list
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8eee59ce/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 6078573..f23b46e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -992,6 +992,12 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.skyscreamer</groupId>
+ <artifactId>jsonassert</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>