You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2014/01/31 04:24:30 UTC
svn commit: r1563042 - in
/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/m...
Author: szetszwo
Date: Fri Jan 31 03:24:24 2014
New Revision: 1563042
URL: http://svn.apache.org/r1563042
Log:
Merge r1555021 through r1563041 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
- copied unchanged from r1563041, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1561511-1563041
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt Fri Jan 31 03:24:24 2014
@@ -82,9 +82,6 @@ Trunk (Unreleased)
BUG FIXES
- MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant.
- (Yu Gao via llu)
-
MAPREDUCE-3194. "mapred mradmin" command is broken in mrv2
(Jason Lowe via bobby)
@@ -130,15 +127,9 @@ Trunk (Unreleased)
MAPREDUCE-4574. Fix TotalOrderParitioner to work with
non-WritableComparable key types. (harsh)
- MAPREDUCE-4884. Streaming tests fail to start MiniMRCluster due to missing
- queue configuration. (Chris Nauroth via suresh)
-
MAPREDUCE-5012. Typo in javadoc for IdentityMapper class. (Adam Monsen
via suresh)
- MAPREDUCE-4885. Streaming tests have multiple failures on Windows. (Chris
- Nauroth via bikas)
-
MAPREDUCE-4987. TestMRJobs#testDistributedCache fails on Windows due to
classpath problems and unexpected behavior of symlinks (Chris Nauroth via
bikas)
@@ -154,6 +145,24 @@ Release 2.4.0 - UNRELEASED
NEW FEATURES
+ IMPROVEMENTS
+
+ MAPREDUCE-5464. Add analogs of the SLOTS_MILLIS counters that jive with the
+ YARN resource model (Sandy Ryza)
+
+ MAPREDUCE-5732. Report proper queue when job has been automatically placed
+ (Sandy Ryza)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+Release 2.3.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
MAPREDUCE-5265. History server admin service to refresh user and superuser
group mappings (Ashwin Shankar via jlowe)
@@ -169,6 +178,19 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5332. Support token-preserving restart of history server (jlowe)
IMPROVEMENTS
+
+ MAPREDUCE-5329. Allow MR applications to use additional AuxServices,
+ which are compatible with the default MapReduce shuffle.
+ (Avner BenHanoch via sseth)
+
+ MAPREDUCE-5463. Deprecate SLOTS_MILLIS counters (Tzuyoshi Ozawa via Sandy
+ Ryza)
+
+ MAPREDUCE-5457. Add a KeyOnlyTextOutputReader to enable streaming to write
+ out text files without separators (Sandy Ryza)
+
+ MAPREDUCE-5596. Allow configuring the number of threads used to serve
+ shuffle connections (Sandy Ryza via jlowe)
MAPREDUCE-434. LocalJobRunner limited to single reducer (Sandy Ryza and
Aaron Kimball via Sandy Ryza)
@@ -208,6 +230,9 @@ Release 2.4.0 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-4680. Job history cleaner should only check timestamps of files in
+ old enough directories (Robert Kanter via Sandy Ryza)
+
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
@@ -218,6 +243,37 @@ Release 2.4.0 - UNRELEASED
BUG FIXES
+ MAPREDUCE-5569. FloatSplitter is not generating correct splits (Nathan
+ Roberts via jlowe)
+
+ MAPREDUCE-5546. mapred.cmd on Windows set HADOOP_OPTS incorrectly (Chuan Liu
+ via cnauroth)
+
+ MAPREDUCE-5518. Fixed typo "can't read paritions file". (Albert Chu
+ via devaraj)
+
+ MAPREDUCE-5561. org.apache.hadoop.mapreduce.v2.app.job.impl.TestJobImpl
+ testcase failing on trunk (Karthik Kambatla via jlowe)
+
+ MAPREDUCE-5598. TestUserDefinedCounters.testMapReduceJob is flakey
+ (Robert Kanter via jlowe)
+
+ MAPREDUCE-5604. TestMRAMWithNonNormalizedCapabilities fails on Windows due to
+ exceeding max path length. (cnauroth)
+
+ MAPREDUCE-5451. MR uses LD_LIBRARY_PATH which doesn't mean anything in
+ Windows. (Yingda Chen via cnauroth)
+
+ MAPREDUCE-5409. MRAppMaster throws InvalidStateTransitonException: Invalid
+ event: TA_TOO_MANY_FETCH_FAILURE at KILLED for TaskAttemptImpl (Gera
+ Shegalov via jlowe)
+
+ MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
+ (Chuan Liu via cnauroth)
+
+ MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params
+ is specified (Gera Shegalov via Sandy Ryza)
+
MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal
task-state (Ashwin Shankar via jlowe)
@@ -291,65 +347,6 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5723. MR AM container log can be truncated or empty.
(Mohammad Kamrul Islam via kasha)
-Release 2.3.0 - UNRELEASED
-
- INCOMPATIBLE CHANGES
-
- NEW FEATURES
-
- IMPROVEMENTS
-
- MAPREDUCE-5329. Allow MR applications to use additional AuxServices,
- which are compatible with the default MapReduce shuffle.
- (Avner BenHanoch via sseth)
-
- MAPREDUCE-5463. Deprecate SLOTS_MILLIS counters (Tzuyoshi Ozawa via Sandy
- Ryza)
-
- MAPREDUCE-5457. Add a KeyOnlyTextOutputReader to enable streaming to write
- out text files without separators (Sandy Ryza)
-
- MAPREDUCE-5596. Allow configuring the number of threads used to serve
- shuffle connections (Sandy Ryza via jlowe)
-
- OPTIMIZATIONS
-
- MAPREDUCE-4680. Job history cleaner should only check timestamps of files in
- old enough directories (Robert Kanter via Sandy Ryza)
-
- BUG FIXES
-
- MAPREDUCE-5569. FloatSplitter is not generating correct splits (Nathan
- Roberts via jlowe)
-
- MAPREDUCE-5546. mapred.cmd on Windows set HADOOP_OPTS incorrectly (Chuan Liu
- via cnauroth)
-
- MAPREDUCE-5518. Fixed typo "can't read paritions file". (Albert Chu
- via devaraj)
-
- MAPREDUCE-5561. org.apache.hadoop.mapreduce.v2.app.job.impl.TestJobImpl
- testcase failing on trunk (Karthik Kambatla via jlowe)
-
- MAPREDUCE-5598. TestUserDefinedCounters.testMapReduceJob is flakey
- (Robert Kanter via jlowe)
-
- MAPREDUCE-5604. TestMRAMWithNonNormalizedCapabilities fails on Windows due to
- exceeding max path length. (cnauroth)
-
- MAPREDUCE-5451. MR uses LD_LIBRARY_PATH which doesn't mean anything in
- Windows. (Yingda Chen via cnauroth)
-
- MAPREDUCE-5409. MRAppMaster throws InvalidStateTransitonException: Invalid
- event: TA_TOO_MANY_FETCH_FAILURE at KILLED for TaskAttemptImpl (Gera
- Shegalov via jlowe)
-
- MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus.
- (Chuan Liu via cnauroth)
-
- MAPREDUCE-5650. Job fails when hprof mapreduce.task.profile.map/reduce.params
- is specified (Gera Shegalov via Sandy Ryza)
-
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
@@ -996,9 +993,15 @@ Release 2.1.0-beta - 2013-08-22
HADOOP-9372. Fix bad timeout annotations on tests.
(Arpit Agarwal via suresh)
+ MAPREDUCE-4885. Streaming tests have multiple failures on Windows. (Chris
+ Nauroth via bikas)
+
MAPREDUCE-5177. Use common utils FileUtil#setReadable/Writable/Executable &
FileUtil#canRead/Write/Execute. (Ivan Mitic via suresh)
+ MAPREDUCE-5349. TestClusterMapReduceTestCase and TestJobName fail on Windows
+ in branch-2. (Chuan Liu via cnauroth)
+
MAPREDUCE-5355. MiniMRYarnCluster with localFs does not work on Windows.
(Chuan Liu via cnauroth)
@@ -1149,6 +1152,9 @@ Release 2.0.3-alpha - 2013-02-06
BUG FIXES
+ MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant.
+ (Yu Gao via llu)
+
MAPREDUCE-4607. Race condition in ReduceTask completion can result in Task
being incorrectly failed. (Bikas Saha via tomwhite)
@@ -1211,6 +1217,9 @@ Release 2.0.3-alpha - 2013-02-06
MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7.
(Arpit Agarwal via suresh)
+ MAPREDUCE-4884. Streaming tests fail to start MiniMRCluster due to missing
+ queue configuration. (Chris Nauroth via suresh)
+
MAPREDUCE-4953. HadoopPipes misuses fprintf. (Andy Isaacson via atm)
Release 2.0.2-alpha - 2012-09-07
@@ -1219,7 +1228,7 @@ Release 2.0.2-alpha - 2012-09-07
NEW FEATURES
- MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process
+ MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process
command-line. (ahmed via tucu)
MAPREDUCE-4417. add support for encrypted shuffle (tucu)
Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1561511-1563041
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri Jan 31 03:24:24 2014
@@ -525,6 +525,12 @@ public class JobHistoryEventHandler exte
JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent();
mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime());
}
+
+ if (event.getHistoryEvent().getEventType() == EventType.JOB_QUEUE_CHANGED) {
+ JobQueueChangeEvent jQueueEvent =
+ (JobQueueChangeEvent) event.getHistoryEvent();
+ mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName());
+ }
// If this is JobFinishedEvent, close the writer and setup the job-index
if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) {
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java Fri Jan 31 03:24:24 2014
@@ -39,7 +39,7 @@ import org.apache.hadoop.security.author
/**
- * Main interface to interact with the job. Provides only getters.
+ * Main interface to interact with the job.
*/
public interface Job {
@@ -98,4 +98,6 @@ public interface Job {
List<AMInfo> getAMInfos();
boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation);
+
+ public void setQueueName(String queueName);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Fri Jan 31 03:24:24 2014
@@ -59,6 +59,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobQueueChangeEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent;
import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
@@ -181,7 +182,7 @@ public class JobImpl implements org.apac
private final EventHandler eventHandler;
private final MRAppMetrics metrics;
private final String userName;
- private final String queueName;
+ private String queueName;
private final long appSubmitTime;
private final AppContext appContext;
@@ -1123,6 +1124,13 @@ public class JobImpl implements org.apac
return queueName;
}
+ @Override
+ public void setQueueName(String queueName) {
+ this.queueName = queueName;
+ JobQueueChangeEvent jqce = new JobQueueChangeEvent(oldJobId, queueName);
+ eventHandler.handle(new JobHistoryEvent(jobId, jqce));
+ }
+
/*
* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.v2.app.job.Job#getConfFile()
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Fri Jan 31 03:24:24 2014
@@ -1265,57 +1265,56 @@ public abstract class TaskAttemptImpl im
}
}
}
-
- private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
+
+ private static void updateMillisCounters(JobCounterUpdateEvent jce,
+ TaskAttemptImpl taskAttempt) {
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
- int slotMemoryReq =
+ long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
+ int mbRequired =
taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
+ int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);
int minSlotMemSize = taskAttempt.conf.getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
int simSlotsRequired =
- minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
+ minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
/ minSlotMemSize);
- long slotMillisIncrement =
- simSlotsRequired
- * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
- return slotMillisIncrement;
+ if (taskType == TaskType.MAP) {
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
+ jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
+ jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
+ jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
+ } else {
+ jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
+ jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
+ jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
+ jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
+ }
}
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded(
TaskAttemptImpl taskAttempt) {
- long slotMillis = computeSlotMillis(taskAttempt);
TaskId taskId = taskAttempt.attemptId.getTaskId();
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
- jce.addCounterUpdate(
- taskId.getTaskType() == TaskType.MAP ?
- JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
- slotMillis);
+ updateMillisCounters(jce, taskAttempt);
return jce;
}
-
+
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
- long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
if (taskType == TaskType.MAP) {
jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
- }
} else {
jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
- }
+ }
+ if (!taskAlreadyCompleted) {
+ updateMillisCounters(jce, taskAttempt);
}
return jce;
}
@@ -1325,20 +1324,13 @@ public abstract class TaskAttemptImpl im
TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
- long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
if (taskType == TaskType.MAP) {
jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
- }
} else {
jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
- if(!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
- }
+ }
+ if (!taskAlreadyCompleted) {
+ updateMillisCounters(jce, taskAttempt);
}
return jce;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java Fri Jan 31 03:24:24 2014
@@ -109,11 +109,11 @@ public abstract class RMCommunicator ext
@Override
protected void serviceStart() throws Exception {
scheduler= createSchedulerProxy();
- register();
- startAllocatorThread();
JobID id = TypeConverter.fromYarn(this.applicationId);
JobId jobId = TypeConverter.toYarn(id);
job = context.getJob(jobId);
+ register();
+ startAllocatorThread();
super.serviceStart();
}
@@ -161,6 +161,9 @@ public abstract class RMCommunicator ext
}
this.applicationACLs = response.getApplicationACLs();
LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
+ String queue = response.getQueue();
+ LOG.info("queue: " + queue);
+ job.setQueueName(queue);
} catch (Exception are) {
LOG.error("Exception while registering", are);
throw new YarnRuntimeException(are);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java Fri Jan 31 03:24:24 2014
@@ -81,6 +81,15 @@ public class TestEvents {
assertEquals(test.getPriority(), JobPriority.LOW);
}
+
+ @Test(timeout = 10000)
+ public void testJobQueueChange() throws Exception {
+ org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
+ JobQueueChangeEvent test = new JobQueueChangeEvent(jid,
+ "newqueue");
+ assertEquals(test.getJobId().toString(), jid.toString());
+ assertEquals(test.getJobQueueName(), "newqueue");
+ }
/**
* simple test TaskUpdatedEvent and TaskUpdated
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Fri Jan 31 03:24:24 2014
@@ -117,6 +117,9 @@ public class MRApp extends MRAppMaster {
private File testWorkDir;
private Path testAbsPath;
private ClusterInfo clusterInfo;
+
+ // Queue to pretend the RM assigned us
+ private String assignedQueue;
public static String NM_HOST = "localhost";
public static int NM_PORT = 1234;
@@ -133,7 +136,7 @@ public class MRApp extends MRAppMaster {
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, Clock clock) {
- this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock);
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock, null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
@@ -146,6 +149,12 @@ public class MRApp extends MRAppMaster {
boolean cleanOnStart) {
this(maps, reduces, autoComplete, testName, cleanOnStart, 1);
}
+
+ public MRApp(int maps, int reduces, boolean autoComplete, String testName,
+ boolean cleanOnStart, String assignedQueue) {
+ this(maps, reduces, autoComplete, testName, cleanOnStart, 1,
+ new SystemClock(), assignedQueue);
+ }
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, boolean unregistered) {
@@ -178,7 +187,7 @@ public class MRApp extends MRAppMaster {
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
- new SystemClock());
+ new SystemClock(), null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
@@ -191,33 +200,34 @@ public class MRApp extends MRAppMaster {
boolean cleanOnStart, int startCount, Clock clock, boolean unregistered) {
this(getApplicationAttemptId(applicationId, startCount), getContainerId(
applicationId, startCount), maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, clock, unregistered);
+ cleanOnStart, startCount, clock, unregistered, null);
}
public MRApp(int maps, int reduces, boolean autoComplete, String testName,
- boolean cleanOnStart, int startCount, Clock clock) {
+ boolean cleanOnStart, int startCount, Clock clock, String assignedQueue) {
this(getApplicationAttemptId(applicationId, startCount), getContainerId(
applicationId, startCount), maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, clock, true);
+ cleanOnStart, startCount, clock, true, assignedQueue);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount, boolean unregistered) {
this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, new SystemClock(), unregistered);
+ cleanOnStart, startCount, new SystemClock(), unregistered, null);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
boolean cleanOnStart, int startCount) {
this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
- cleanOnStart, startCount, new SystemClock(), true);
+ cleanOnStart, startCount, new SystemClock(), true, null);
}
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
int maps, int reduces, boolean autoComplete, String testName,
- boolean cleanOnStart, int startCount, Clock clock, boolean unregistered) {
+ boolean cleanOnStart, int startCount, Clock clock, boolean unregistered,
+ String assignedQueue) {
super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System
.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
this.testWorkDir = new File("target", testName);
@@ -239,6 +249,7 @@ public class MRApp extends MRAppMaster {
// If safeToReportTerminationToUser is set to true, we can verify whether
// the job can reaches the final state when MRAppMaster shuts down.
this.successfullyUnregistered.set(unregistered);
+ this.assignedQueue = assignedQueue;
}
@Override
@@ -285,6 +296,9 @@ public class MRApp extends MRAppMaster {
start();
DefaultMetricsSystem.shutdown();
Job job = getContext().getAllJobs().values().iterator().next();
+ if (assignedQueue != null) {
+ job.setQueueName(assignedQueue);
+ }
// Write job.xml
String jobFile = MRApps.getJobFile(conf, user,
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java Fri Jan 31 03:24:24 2014
@@ -39,6 +39,7 @@ public class MockAppContext implements A
final Map<JobId, Job> jobs;
final long startTime = System.currentTimeMillis();
Set<String> blacklistedNodes;
+ String queue;
public MockAppContext(int appid) {
appID = MockJobs.newAppID(appid);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Fri Jan 31 03:24:24 2014
@@ -629,6 +629,11 @@ public class MockJobs extends MockApps {
jobConf.addResource(fc.open(configFile), configFile.toString());
return jobConf;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ // do nothing
+ }
};
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java Fri Jan 31 03:24:24 2014
@@ -37,7 +37,7 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -199,7 +199,7 @@ public class TestJobEndNotifier extends
@Test
public void testNotificationOnLastRetryNormalShutdown() throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
// Act like it is the second attempt. Default max attempts is 2
MRApp app = spy(new MRAppWithCustomContainerAllocator(
2, 2, true, this.getClass().getName(), true, 2, true));
@@ -223,7 +223,7 @@ public class TestJobEndNotifier extends
@Test
public void testAbsentNotificationOnNotLastRetryUnregistrationFailure()
throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
MRApp app = spy(new MRAppWithCustomContainerAllocator(2, 2, false,
this.getClass().getName(), true, 1, false));
doNothing().when(app).sysexit();
@@ -250,7 +250,7 @@ public class TestJobEndNotifier extends
@Test
public void testNotificationOnLastRetryUnregistrationFailure()
throws Exception {
- HttpServer server = startHttpServer();
+ HttpServer2 server = startHttpServer();
MRApp app = spy(new MRAppWithCustomContainerAllocator(2, 2, false,
this.getClass().getName(), true, 2, false));
doNothing().when(app).sysexit();
@@ -274,10 +274,10 @@ public class TestJobEndNotifier extends
server.stop();
}
- private static HttpServer startHttpServer() throws Exception {
+ private static HttpServer2 startHttpServer() throws Exception {
new File(System.getProperty(
"build.webapps", "build/webapps") + "/test").mkdirs();
- HttpServer server = new HttpServer.Builder().setName("test")
+ HttpServer2 server = new HttpServer2.Builder().setName("test")
.addEndpoint(URI.create("http://localhost:0"))
.setFindPort(true).build();
server.addServlet("jobend", "/jobend", JobEndServlet.class);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Fri Jan 31 03:24:24 2014
@@ -505,6 +505,11 @@ public class TestRuntimeEstimators {
public Configuration loadConfFile() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ // do nothing
+ }
}
/*
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Fri Jan 31 03:24:24 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapTaskAttemptImpl;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
@@ -182,13 +183,13 @@ public class TestTaskAttempt{
}
@Test
- public void testSlotMillisCounterUpdate() throws Exception {
- verifySlotMillis(2048, 2048, 1024);
- verifySlotMillis(2048, 1024, 1024);
- verifySlotMillis(10240, 1024, 2048);
+ public void testMillisCountersUpdate() throws Exception {
+ verifyMillisCounters(2048, 2048, 1024);
+ verifyMillisCounters(2048, 1024, 1024);
+ verifyMillisCounters(10240, 1024, 2048);
}
- public void verifySlotMillis(int mapMemMb, int reduceMemMb,
+ public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
int minContainerSize) throws Exception {
Clock actualClock = new SystemClock();
ControlledClock clock = new ControlledClock(actualClock);
@@ -232,13 +233,23 @@ public class TestTaskAttempt{
Assert.assertEquals(mta.getLaunchTime(), 10);
Assert.assertEquals(rta.getFinishTime(), 11);
Assert.assertEquals(rta.getLaunchTime(), 10);
+ Counters counters = job.getAllCounters();
Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
- job.getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_MAPS)
- .getValue());
- Assert.assertEquals(
- (int) Math.ceil((float) reduceMemMb / minContainerSize), job
- .getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_REDUCES)
- .getValue());
+ counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
+ Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
+ counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
+ Assert.assertEquals(mapMemMb,
+ counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
+ Assert.assertEquals(reduceMemMb,
+ counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
+ Assert.assertEquals(1,
+ counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
private TaskAttemptImpl createMapTaskAttemptImplForTest(
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr Fri Jan 31 03:24:24 2014
@@ -122,6 +122,13 @@
]
},
+ {"type": "record", "name": "JobQueueChange",
+ "fields": [
+ {"name": "jobid", "type": "string"},
+ {"name": "jobQueueName", "type": "string"}
+ ]
+ },
+
{"type": "record", "name": "JobUnsuccessfulCompletion",
"fields": [
{"name": "jobid", "type": "string"},
@@ -267,6 +274,7 @@
"JOB_FINISHED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
+ "JOB_QUEUE_CHANGED",
"JOB_FAILED",
"JOB_KILLED",
"JOB_ERROR",
@@ -306,6 +314,7 @@
"JobInited",
"AMStarted",
"JobPriorityChange",
+ "JobQueueChange",
"JobStatusChanged",
"JobSubmitted",
"JobUnsuccessfulCompletion",
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java Fri Jan 31 03:24:24 2014
@@ -49,5 +49,11 @@ public enum JobCounter {
TASKS_REQ_PREEMPT,
CHECKPOINTS,
CHECKPOINT_BYTES,
- CHECKPOINT_TIME
+ CHECKPOINT_TIME,
+ MILLIS_MAPS,
+ MILLIS_REDUCES,
+ VCORES_MILLIS_MAPS,
+ VCORES_MILLIS_REDUCES,
+ MB_MILLIS_MAPS,
+ MB_MILLIS_REDUCES
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java Fri Jan 31 03:24:24 2014
@@ -98,6 +98,8 @@ public class EventReader implements Clos
result = new JobFinishedEvent(); break;
case JOB_PRIORITY_CHANGED:
result = new JobPriorityChangeEvent(); break;
+ case JOB_QUEUE_CHANGED:
+ result = new JobQueueChangeEvent(); break;
case JOB_STATUS_CHANGED:
result = new JobStatusChangedEvent(); break;
case JOB_FAILED:
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java Fri Jan 31 03:24:24 2014
@@ -183,6 +183,9 @@ public class JobHistoryParser implements
case JOB_PRIORITY_CHANGED:
handleJobPriorityChangeEvent((JobPriorityChangeEvent) event);
break;
+ case JOB_QUEUE_CHANGED:
+ handleJobQueueChangeEvent((JobQueueChangeEvent) event);
+ break;
case JOB_FAILED:
case JOB_KILLED:
case JOB_ERROR:
@@ -385,6 +388,10 @@ public class JobHistoryParser implements
private void handleJobPriorityChangeEvent(JobPriorityChangeEvent event) {
info.priority = event.getPriority();
}
+
+ private void handleJobQueueChangeEvent(JobQueueChangeEvent event) {
+ info.jobQueueName = event.getJobQueueName();
+ }
private void handleJobInitedEvent(JobInitedEvent event) {
info.launchTime = event.getLaunchTime();
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties Fri Jan 31 03:24:24 2014
@@ -25,9 +25,15 @@ DATA_LOCAL_MAPS.name= Data-
RACK_LOCAL_MAPS.name= Rack-local map tasks
SLOTS_MILLIS_MAPS.name= Total time spent by all maps in occupied slots (ms)
SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms)
+MILLIS_MAPS.name= Total time spent by all map tasks (ms)
+MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms)
+MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map tasks
+MB_MILLIS_REDUCES.name= Total megabyte-seconds taken by all reduce tasks
+VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks
+VCORES_MILLIS_REDUCES.name= Total vcore-seconds taken by all reduce tasks
FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms)
FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms)
TASKS_REQ_PREEMPT.name= Tasks that have been asked to preempt
CHECKPOINTS.name= Number of checkpoints reported
CHECKPOINT_BYTES.name= Total amount of bytes in checkpoints
-CHECKPOINT_TIME.name= Total time spent checkpointing (ms)
\ No newline at end of file
+CHECKPOINT_TIME.name= Total time spent checkpointing (ms)
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java Fri Jan 31 03:24:24 2014
@@ -34,10 +34,10 @@ import javax.servlet.http.HttpServletRes
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
public class TestJobEndNotifier extends TestCase {
- HttpServer server;
+ HttpServer2 server;
URL baseUrl;
@SuppressWarnings("serial")
@@ -102,7 +102,7 @@ public class TestJobEndNotifier extends
public void setUp() throws Exception {
new File(System.getProperty("build.webapps", "build/webapps") + "/test"
).mkdirs();
- server = new HttpServer.Builder().setName("test")
+ server = new HttpServer2.Builder().setName("test")
.addEndpoint(URI.create("http://localhost:0"))
.setFindPort(true).build();
server.addServlet("delay", "/delay", DelayServlet.class);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Fri Jan 31 03:24:24 2014
@@ -453,4 +453,9 @@ public class CompletedJob implements org
}
return amInfos;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ throw new UnsupportedOperationException("Can't set job's queue name in history");
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java Fri Jan 31 03:24:24 2014
@@ -190,5 +190,10 @@ public class PartialJob implements org.a
public List<AMInfo> getAMInfos() {
return null;
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ throw new UnsupportedOperationException("Can't set job's queue name in history");
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java Fri Jan 31 03:24:24 2014
@@ -155,6 +155,41 @@ public class TestJobHistoryEvents {
Assert.assertEquals("JobHistoryEventHandler",
services[services.length - 1].getName());
}
+
+ @Test
+ public void testAssignedQueue() throws Exception {
+ Configuration conf = new Configuration();
+ MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(),
+ true, "assignedQueue");
+ app.submit(conf);
+ Job job = app.getContext().getAllJobs().values().iterator().next();
+ JobId jobId = job.getID();
+ LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
+ app.waitForState(job, JobState.SUCCEEDED);
+
+ //make sure all events are flushed
+ app.waitForState(Service.STATE.STOPPED);
+ /*
+ * Use HistoryContext to read logged events and verify the number of
+ * completed maps
+ */
+ HistoryContext context = new JobHistory();
+ // test start and stop states
+ ((JobHistory)context).init(conf);
+ ((JobHistory)context).start();
+ Assert.assertTrue( context.getStartTime()>0);
+ Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);
+
+ // get job before stopping JobHistory
+ Job parsedJob = context.getJob(jobId);
+
+ // stop JobHistory
+ ((JobHistory)context).stop();
+ Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);
+
+ Assert.assertEquals("QueueName not correct", "assignedQueue",
+ parsedJob.getQueueName());
+ }
private void verifyTask(Task task) {
Assert.assertEquals("Task state not currect", TaskState.SUCCEEDED,
@@ -184,6 +219,11 @@ public class TestJobHistoryEvents {
super(maps, reduces, autoComplete, testName, cleanOnStart);
}
+ public MRAppWithHistory(int maps, int reduces, boolean autoComplete,
+ String testName, boolean cleanOnStart, String assignedQueue) {
+ super(maps, reduces, autoComplete, testName, cleanOnStart, assignedQueue);
+ }
+
@Override
protected EventHandler<JobHistoryEvent> createJobHistoryHandler(
AppContext context) {
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java Fri Jan 31 03:24:24 2014
@@ -415,5 +415,9 @@ public class TestHsWebServicesAcls {
return aclsMgr.checkAccess(callerUGI, jobOperation,
this.getUserName(), jobAcls.get(jobOperation));
}
+
+ @Override
+ public void setQueueName(String queueName) {
+ }
}
}