You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/11/25 22:03:53 UTC
hadoop git commit: MAPREDUCE-5568. Fixed CompletedJob in JHS to show
progress percentage correctly in case the number of mappers or reducers is
zero. Contributed by MinJi Kim (cherry picked from commit
78f7cdbfd6e2b9fac51c369c748ae93d12ef065a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 286305653 -> 80e9be2d9
MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage correctly in case the number of mappers or reducers is zero. Contributed by MinJi Kim
(cherry picked from commit 78f7cdbfd6e2b9fac51c369c748ae93d12ef065a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80e9be2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80e9be2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80e9be2d
Branch: refs/heads/branch-2
Commit: 80e9be2d956ce6e5f6dee4418c7a83c64bae3818
Parents: 2863056
Author: Jian He <ji...@apache.org>
Authored: Tue Nov 25 13:02:55 2014 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Nov 25 13:03:40 2014 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 +++
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 14 +++++++--
.../mapreduce/v2/hs/TestJobHistoryEntities.java | 30 ++++++++++++++++----
...33-2-0-SUCCEEDED-default-1416424779349.jhist | 29 +++++++++++++++++++
4 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e9be2d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 06e5c44..8223777 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -23,6 +23,10 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6049. AM JVM does not exit if MRClientService graceful shutdown
fails (Rohith via devaraj)
+
+ MAPREDUCE-5568. Fixed CompletedJob in JHS to show progress percentage
+ correctly in case the number of mappers or reducers is zero. (MinJi Kim via
+ jianhe)
Release 2.6.0 - 2014-11-18
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e9be2d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index 9e38316..66c44c8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -141,8 +141,18 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
report.setFinishTime(jobInfo.getFinishTime());
report.setJobName(jobInfo.getJobname());
report.setUser(jobInfo.getUsername());
- report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
- report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
+
+ if ( getTotalMaps() == 0 ) {
+ report.setMapProgress(1.0f);
+ } else {
+ report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
+ }
+ if ( getTotalReduces() == 0 ) {
+ report.setReduceProgress(1.0f);
+ } else {
+ report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
+ }
+
report.setJobFile(getConfFile().toString());
String historyUrl = "N/A";
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e9be2d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
index e96f228..b1f90e6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
@@ -55,14 +55,19 @@ public class TestJobHistoryEntities {
private final String historyFileName =
"job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
+ private final String historyFileNameZeroReduceTasks =
+ "job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist";
private final String confFileName = "job_1329348432655_0001_conf.xml";
private final Configuration conf = new Configuration();
private final JobACLsManager jobAclsManager = new JobACLsManager(conf);
private boolean loadTasks;
private JobId jobId = MRBuilderUtils.newJobId(1329348432655l, 1, 1);
- Path fulleHistoryPath =
+ Path fullHistoryPath =
new Path(this.getClass().getClassLoader().getResource(historyFileName)
.getFile());
+ Path fullHistoryPathZeroReduces =
+ new Path(this.getClass().getClassLoader().getResource(historyFileNameZeroReduceTasks)
+ .getFile());
Path fullConfPath =
new Path(this.getClass().getClassLoader().getResource(confFileName)
.getFile());
@@ -87,7 +92,7 @@ public class TestJobHistoryEntities {
when(info.getConfFile()).thenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob =
- new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+ new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
assertEquals(loadTasks, completedJob.tasksLoaded.get());
@@ -106,12 +111,27 @@ public class TestJobHistoryEntities {
assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
+ @Test (timeout=100000)
+ public void testCopmletedJobReportWithZeroTasks() throws Exception {
+ HistoryFileInfo info = mock(HistoryFileInfo.class);
+ when(info.getConfFile()).thenReturn(fullConfPath);
+ completedJob =
+ new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks, "user",
+ info, jobAclsManager);
+ JobReport jobReport = completedJob.getReport();
+ // Make sure that the number reduces (completed and total) are equal to zero.
+ assertEquals(0, completedJob.getTotalReduces());
+ assertEquals(0, completedJob.getCompletedReduces());
+ // Verify that the reduce progress is 1.0 (not NaN)
+ assertEquals(1.0, jobReport.getReduceProgress(), 0.001);
+ }
+
@Test (timeout=10000)
public void testCompletedTask() throws Exception {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
- new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+ new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@@ -140,7 +160,7 @@ public class TestJobHistoryEntities {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
- new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+ new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
@@ -179,7 +199,7 @@ public class TestJobHistoryEntities {
HistoryFileInfo info = mock(HistoryFileInfo.class);
when(info.getConfFile()).thenReturn(fullConfPath);
completedJob =
- new CompletedJob(conf, jobId, fulleHistoryPath, loadTasks, "user",
+ new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskCompletionEvent[] events= completedJob.getMapAttemptCompletionEvents(0,1000);
assertEquals(10, completedJob.getMapAttemptCompletionEvents(0,10).length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80e9be2d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
new file mode 100644
index 0000000..1a02fc8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1416424547277_0002-1416424775281-root-TeraGen-1416424785433-2-0-SUCCEEDED-default-1416424779349.jhist
@@ -0,0 +1,29 @@
+Avro-Json
+{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"f
inishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"t
otalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"st
ring"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finis
hTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}}
,{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type"
:["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"stri
ng"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]}
+{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1416424547277_0002_000001","startTime":1416424776369,"containerId":"container_1416424547277_0002_01_000001","nodeManagerHost":"mfs137.qa.lab","nodeManagerPort":35535,"nodeManagerHttpPort":8042}}}
+
+{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1416424547277_0002","jobName":"TeraGen","userName":"root","submitTime":1416424775281,"jobConfPath":"hdfs://localhost:9000/tmp/hadoop-yarn/staging/root/.staging/job_1416424547277_0002/job.xml","acls":{},"jobQueueName":"default","workflowId":{"string":""},"workflowName":{"string":""},"workflowNodeName":{"string":""},"workflowAdjacencies":{"string":""},"workflowTags":{"string":""}}}}
+
+{"type":"JOB_QUEUE_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobQueueChange":{"jobid":"job_1416424547277_0002","jobQueueName":"default"}}}
+
+{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1416424547277_0002","launchTime":1416424779349,"totalMaps":2,"totalReduces":0,"jobStatus":"INITED","uberized":false}}}
+
+{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1416424547277_0002","submitTime":1416424775281,"launchTime":1416424779349}}}
+
+{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","startTime":1416424779381,"splitLocations":""}}}
+
+{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","startTime":1416424779383,"splitLocations":""}}}
+
+{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","attemptId":"attempt_1416424547277_0002_m_000000_0","startTime":1416424781561,"trackerName":"mfs137.qa.lab","httpPort":8042,"shufflePort":13562,"containerId":"container_1416424547277_0002_01_000002","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}}
+
+{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","attemptId":"attempt_1416424547277_0002_m_000001_0","startTime":1416424782404,"trackerName":"mfs137.qa.lab","httpPort":8042,"shufflePort":13562,"containerId":"container_1416424547277_0002_01_000003","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}}
+
+{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1416424547277_0002_m_000000","attemptId":"attempt_1416424547277_0002_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1416424784304,"finishTime":1416424784542,"hostname":"mfs137.qa.lab","port":35535,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDF
S: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":36},{
"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":550},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":142049280},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":118655541199}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"clockSplits":[2772,18,1
8,18,18,18,18,18,18,18,18,18],"cpuUsages":[45,46,46,46,46,46,45,46,46,46,46,46],"vMemKbytes":[47585,142755,237925,333094,428265,523434,618605,713774,808945,904115,999284,1094455],"physMemKbytes":[5780,17340,28900,40459,52020,63579,75140,86699,98260,109820,121379,132940]}}}
+
+{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1416424547277_0002_m_000000","taskType":"MAP","finishTime":1416424784542,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of
read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":36},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":550},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":142049280},{"nam
e":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":118655541199}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"successfulAttemptId":{"string":"attempt_1416424547277_0002_m_000000_0"}}}}
+
+{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1416424547277_0002_m_000001","attemptId":"attempt_1416424547277_0002_m_000001_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1416424785165,"finishTime":1416424785379,"hostname":"mfs137.qa.lab","port":35535,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDF
S: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":34},{
"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":510},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":142311424},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":114863641618}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"clockSplits":[2781,17,1
7,17,17,18,17,17,17,17,17,18],"cpuUsages":[42,43,42,43,42,43,42,43,42,43,42,43],"vMemKbytes":[47585,142755,237925,333094,428265,523434,618605,713774,808945,904115,999284,1094455],"physMemKbytes":[5790,17371,28953,40534,52115,63696,75278,86859,98441,110022,121603,133185]}}}
+
+{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1416424547277_0002_m_000001","taskType":"MAP","finishTime":1416424785379,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":96586},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":79},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":5000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of
read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":2}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":50},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":50},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":79},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":34},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":510},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":142311424},{"nam
e":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":1169448960},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":152043520}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":114863641618}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":5000}]}]},"successfulAttemptId":{"string":"attempt_1416424547277_0002_m_000001_0"}}}}
+
+{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1416424547277_0002","finishTime":1416424785433,"finishedMaps":2,"finishedReduces":0,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":193172},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":158},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":10000},{"name":"HDFS_READ_OP
S","displayName":"HDFS: Number of read operations","value":8},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":4}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":2},{"name":"OTHER_LOCAL_MAPS","displayName":"Other local map tasks","value":2},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps in occupied slots (ms)","value":5956},{"name":"MILLIS_MAPS","displayName":"Total time spent by all map tasks (ms)","value":5956},{"name":"VCORES_MILLIS_MAPS","displayName":"Total vcore-seconds taken by all map tasks","value":5956},{"name":"MB_MILLIS_MAPS","displayName":"Total megabyte-seconds taken by all map tasks","value":6098944}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name
":"MAP_INPUT_RECORDS","displayName":"Map input records","value":100},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":100},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":158},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":70},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":1060},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":284360704},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2338897920},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":304087040}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort
.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":233519182817}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":10000}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":193172},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_
WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":158},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":10000},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":8},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":4}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":100},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":100},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":158},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":0},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","
displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":70},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":1060},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":284360704},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":2338897920},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":304087040}]},{"name":"org.apache.hadoop.examples.terasort.TeraGen$Counters","displayName":"org.apache.hadoop.examples.terasort.TeraGen$Counters","counts":[{"name":"CHECKSUM","displayName":"CHECKSUM","value":233519182817}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output For
mat Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":10000}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[]}}}}