You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2013/02/26 01:10:19 UTC
svn commit: r1449976 - in
/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-clien...
Author: szetszwo
Date: Tue Feb 26 00:10:17 2013
New Revision: 1449976
URL: http://svn.apache.org/r1449976
Log:
Merge r1448505 through r1449957 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1448505-1449957
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Tue Feb 26 00:10:17 2013
@@ -177,6 +177,15 @@ Release 2.0.4-beta - UNRELEASED
MAPREDUCE-4994. Addendum fixing testcases failures. (sandyr via tucu)
+ MAPREDUCE-4846. Some JobQueueInfo methods are public in MR1 but protected
+ in MR2. (Sandy Ryza via tomwhite)
+
+ MAPREDUCE-5013. mapred.JobStatus compatibility: MR2 missing constructors
+ from MR1. (Sandy Ryza via tomwhite)
+
+ MAPREDUCE-4951. Container preemption interpreted as task failure.
+ (Sandy Ryza via tomwhite)
+
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1448505-1449957
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1448505-1449957
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Feb 26 00:10:17 2013
@@ -238,7 +238,6 @@ public abstract class TaskAttemptImpl im
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
CLEANUP_CONTAINER_TRANSITION)
- // ^ If RM kills the container due to expiry, preemption etc.
.addTransition(TaskAttemptStateInternal.ASSIGNED,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Tue Feb 26 00:10:17 2013
@@ -67,9 +67,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Allocates the container from the ResourceManager scheduler.
*/
@@ -606,8 +609,8 @@ public class RMContainerAllocator extend
assignedRequests.remove(attemptID);
// send the container completed event to Task attempt
- eventHandler.handle(new TaskAttemptEvent(attemptID,
- TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+ eventHandler.handle(createContainerFinishedEvent(cont, attemptID));
+
// Send the diagnostics
String diagnostics = StringInterner.weakIntern(cont.getDiagnostics());
eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
@@ -617,6 +620,19 @@ public class RMContainerAllocator extend
return newContainers;
}
+ @VisibleForTesting
+ public TaskAttemptEvent createContainerFinishedEvent(ContainerStatus cont,
+ TaskAttemptId attemptID) {
+ if (cont.getExitStatus() == YarnConfiguration.ABORTED_CONTAINER_EXIT_STATUS) {
+ // killed by framework
+ return new TaskAttemptEvent(attemptID,
+ TaskAttemptEventType.TA_KILL);
+ } else {
+ return new TaskAttemptEvent(attemptID,
+ TaskAttemptEventType.TA_CONTAINER_COMPLETED);
+ }
+ }
+
@SuppressWarnings("unchecked")
private void handleUpdatedNodes(AMResponse response) {
// send event to the job about on updated nodes
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Tue Feb 26 00:10:17 2013
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.Event;
@@ -1645,6 +1646,32 @@ public class TestRMContainerAllocator {
Assert.assertTrue(callbackCalled.get());
}
+ @Test
+ public void testCompletedContainerEvent() {
+ RMContainerAllocator allocator = new RMContainerAllocator(
+ mock(ClientService.class), mock(AppContext.class));
+
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(
+ MRBuilderUtils.newTaskId(
+ MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
+ ContainerId containerId = BuilderUtils.newContainerId(1, 1, 1, 1);
+ ContainerStatus status = BuilderUtils.newContainerStatus(
+ containerId, ContainerState.RUNNING, "", 0);
+
+ ContainerStatus abortedStatus = BuilderUtils.newContainerStatus(
+ containerId, ContainerState.RUNNING, "",
+ YarnConfiguration.ABORTED_CONTAINER_EXIT_STATUS);
+
+ TaskAttemptEvent event = allocator.createContainerFinishedEvent(status,
+ attemptId);
+ Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED,
+ event.getType());
+
+ TaskAttemptEvent abortedEvent = allocator.createContainerFinishedEvent(
+ abortedStatus, attemptId);
+ Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent.getType());
+ }
+
public static void main(String[] args) throws Exception {
TestRMContainerAllocator t = new TestRMContainerAllocator();
t.testSimple();
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueInfo.java Tue Feb 26 00:10:17 2013
@@ -67,7 +67,8 @@ public class JobQueueInfo extends QueueI
*
* @param queueName Name of the job queue.
*/
- protected void setQueueName(String queueName) {
+ @InterfaceAudience.Private
+ public void setQueueName(String queueName) {
super.setQueueName(queueName);
}
@@ -76,7 +77,8 @@ public class JobQueueInfo extends QueueI
*
* @param schedulingInfo
*/
- protected void setSchedulingInfo(String schedulingInfo) {
+ @InterfaceAudience.Private
+ public void setSchedulingInfo(String schedulingInfo) {
super.setSchedulingInfo(schedulingInfo);
}
@@ -84,15 +86,21 @@ public class JobQueueInfo extends QueueI
* Set the state of the queue
* @param state state of the queue.
*/
- protected void setQueueState(String state) {
+ @InterfaceAudience.Private
+ public void setQueueState(String state) {
super.setState(QueueState.getState(state));
}
- String getQueueState() {
+ /**
+ * Use getState() instead
+ */
+ @Deprecated
+ public String getQueueState() {
return super.getState().toString();
}
- protected void setChildren(List<JobQueueInfo> children) {
+ @InterfaceAudience.Private
+ public void setChildren(List<JobQueueInfo> children) {
List<QueueInfo> list = new ArrayList<QueueInfo>();
for (JobQueueInfo q : children) {
list.add(q);
@@ -108,7 +116,8 @@ public class JobQueueInfo extends QueueI
return list;
}
- protected void setProperties(Properties props) {
+ @InterfaceAudience.Private
+ public void setProperties(Properties props) {
super.setProperties(props);
}
@@ -141,7 +150,8 @@ public class JobQueueInfo extends QueueI
setChildren(children);
}
- protected void setJobStatuses(org.apache.hadoop.mapreduce.JobStatus[] stats) {
+ @InterfaceAudience.Private
+ public void setJobStatuses(org.apache.hadoop.mapreduce.JobStatus[] stats) {
super.setJobStatuses(stats);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java Tue Feb 26 00:10:17 2013
@@ -77,6 +77,59 @@ public class JobStatus extends org.apach
*/
public JobStatus() {
}
+
+ @Deprecated
+ public JobStatus(JobID jobid, float mapProgress, float reduceProgress,
+ float cleanupProgress, int runState) {
+ this(jobid, mapProgress, reduceProgress, cleanupProgress, runState, null,
+ null, null, null);
+ }
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param runState The current state of the job
+ */
+ @Deprecated
+ public JobStatus(JobID jobid, float mapProgress, float reduceProgress,
+ int runState) {
+ this (jobid, mapProgress, reduceProgress, runState, null, null, null, null);
+ }
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ */
+ @Deprecated
+ public JobStatus(JobID jobid, float mapProgress, float reduceProgress,
+ float cleanupProgress, int runState, JobPriority jp) {
+ this(jobid, mapProgress, reduceProgress, cleanupProgress, runState, jp,
+ null, null, null, null);
+ }
+
+ /**
+ * Create a job status object for a given jobid.
+ * @param jobid The jobid of the job
+ * @param setupProgress The progress made on the setup
+ * @param mapProgress The progress made on the maps
+ * @param reduceProgress The progress made on the reduces
+ * @param cleanupProgress The progress made on the cleanup
+ * @param runState The current state of the job
+ * @param jp Priority of the job.
+ */
+ @Deprecated
+ public JobStatus(JobID jobid, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress,
+ int runState, JobPriority jp) {
+ this(jobid, setupProgress, mapProgress, reduceProgress, cleanupProgress,
+ runState, jp, null, null, null, null);
+ }
/**
* Create a job status object for a given jobid.
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java?rev=1449976&r1=1449975&r2=1449976&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java Tue Feb 26 00:10:17 2013
@@ -449,7 +449,7 @@ class QueueConfigurationParser {
q.appendChild(propsElement);
// Queue-state
- String queueState = jqi.getQueueState();
+ String queueState = jqi.getState().getStateName();
if (queueState != null
&& !queueState.equals(QueueState.UNDEFINED.getStateName())) {
Element qStateElement = document.createElement(STATE_TAG);
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1448505-1449957