You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2015/08/14 21:29:57 UTC
hadoop git commit: MAPREDUCE-5817. Mappers get rescheduled on node
transition even after all reducers are completed. (Sangjin Lee via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 84bf71295 -> 27d24f96a
MAPREDUCE-5817. Mappers get rescheduled on node transition even after all reducers are completed. (Sangjin Lee via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27d24f96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27d24f96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27d24f96
Branch: refs/heads/trunk
Commit: 27d24f96ab8d17e839a1ef0d7076efc78d28724a
Parents: 84bf712
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri Aug 14 12:29:50 2015 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Fri Aug 14 12:29:50 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 38 ++++--
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 130 ++++++++++++++++++-
3 files changed, 156 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d24f96/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c424667..19bd697 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -552,6 +552,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6433. launchTime may be negative. (Zhihai Xu)
+ MAPREDUCE-5817. Mappers get rescheduled on node transition even after all
+ reducers are completed. (Sangjin Lee via kasha)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d24f96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 9d141eb..fc9a3a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -129,6 +129,7 @@ import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/** Implementation of Job interface. Maintains the state machines of Job.
@@ -1330,15 +1331,21 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
// rerun previously successful map tasks
- List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
- if(taskAttemptIdList != null) {
- String mesg = "TaskAttempt killed because it ran on unusable node "
- + nodeId;
- for(TaskAttemptId id : taskAttemptIdList) {
- if(TaskType.MAP == id.getTaskId().getTaskType()) {
- // reschedule only map tasks because their outputs maybe unusable
- LOG.info(mesg + ". AttemptId:" + id);
- eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
+ // do this only if the job is still in the running state and there are
+ // running reducers
+ if (getInternalState() == JobStateInternal.RUNNING &&
+ !allReducersComplete()) {
+ List<TaskAttemptId> taskAttemptIdList =
+ nodesToSucceededTaskAttempts.get(nodeId);
+ if (taskAttemptIdList != null) {
+ String mesg = "TaskAttempt killed because it ran on unusable node "
+ + nodeId;
+ for (TaskAttemptId id : taskAttemptIdList) {
+ if (TaskType.MAP == id.getTaskId().getTaskType()) {
+ // reschedule only map tasks because their outputs maybe unusable
+ LOG.info(mesg + ". AttemptId:" + id);
+ eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
+ }
}
}
}
@@ -1346,6 +1353,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
// RMContainerAllocator
}
+ private boolean allReducersComplete() {
+ return numReduceTasks == 0 || numReduceTasks == getCompletedReduces();
+ }
+
/*
private int getBlockSize() {
String inputClassName = conf.get(MRJobConfig.INPUT_FORMAT_CLASS_ATTR);
@@ -2084,13 +2095,18 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
}
}
+ @VisibleForTesting
+ void decrementSucceededMapperCount() {
+ completedTaskCount--;
+ succeededMapTaskCount--;
+ }
+
private static class MapTaskRescheduledTransition implements
SingleArcTransition<JobImpl, JobEvent> {
@Override
public void transition(JobImpl job, JobEvent event) {
//succeeded map task is restarted back
- job.completedTaskCount--;
- job.succeededMapTaskCount--;
+ job.decrementSucceededMapperCount();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d24f96/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
index cae9663..2af4380 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
@@ -19,15 +19,21 @@
package org.apache.hadoop.mapreduce.v2.app.job.impl;
import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
@@ -35,10 +41,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.jobhistory.EventType;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus.State;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -46,10 +48,17 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.EventType;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
@@ -64,7 +73,11 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobStartEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
@@ -76,6 +89,9 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -508,6 +524,112 @@ public class TestJobImpl {
commitHandler.stop();
}
+ @Test(timeout=20000)
+ public void testUnusableNodeTransition() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
+ conf.setInt(MRJobConfig.NUM_REDUCES, 1);
+ AsyncDispatcher dispatcher = new AsyncDispatcher();
+ dispatcher.init(conf);
+ dispatcher.start();
+ CyclicBarrier syncBarrier = new CyclicBarrier(2);
+ OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true);
+ CommitterEventHandler commitHandler =
+ createCommitterEventHandler(dispatcher, committer);
+ commitHandler.init(conf);
+ commitHandler.start();
+
+ final JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
+ // add a special task event handler to put the task back to running in case
+ // of task rescheduling/killing
+ EventHandler<TaskAttemptEvent> taskAttemptEventHandler =
+ new EventHandler<TaskAttemptEvent>() {
+ @Override
+ public void handle(TaskAttemptEvent event) {
+ if (event.getType() == TaskAttemptEventType.TA_KILL) {
+ job.decrementSucceededMapperCount();
+ }
+ }
+ };
+ dispatcher.register(TaskAttemptEventType.class, taskAttemptEventHandler);
+
+ // replace the tasks with spied versions to return the right attempts
+ Map<TaskId,Task> spiedTasks = new HashMap<TaskId,Task>();
+ List<NodeReport> nodeReports = new ArrayList<NodeReport>();
+ Map<NodeReport,TaskId> nodeReportsToTaskIds =
+ new HashMap<NodeReport,TaskId>();
+ for (Map.Entry<TaskId,Task> e: job.tasks.entrySet()) {
+ TaskId taskId = e.getKey();
+ Task task = e.getValue();
+ if (taskId.getTaskType() == TaskType.MAP) {
+ // add an attempt to the task to simulate nodes
+ NodeId nodeId = mock(NodeId.class);
+ TaskAttempt attempt = mock(TaskAttempt.class);
+ when(attempt.getNodeId()).thenReturn(nodeId);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ when(attempt.getID()).thenReturn(attemptId);
+ // create a spied task
+ Task spied = spy(task);
+ doReturn(attempt).when(spied).getAttempt(any(TaskAttemptId.class));
+ spiedTasks.put(taskId, spied);
+
+ // create a NodeReport based on the node id
+ NodeReport report = mock(NodeReport.class);
+ when(report.getNodeState()).thenReturn(NodeState.UNHEALTHY);
+ when(report.getNodeId()).thenReturn(nodeId);
+ nodeReports.add(report);
+ nodeReportsToTaskIds.put(report, taskId);
+ }
+ }
+ // replace the tasks with the spied tasks
+ job.tasks.putAll(spiedTasks);
+
+ // complete all mappers first
+ for (TaskId taskId: job.tasks.keySet()) {
+ if (taskId.getTaskType() == TaskType.MAP) {
+ // generate a task attempt completed event first to populate the
+ // nodes-to-succeeded-attempts map
+ TaskAttemptCompletionEvent tce =
+ Records.newRecord(TaskAttemptCompletionEvent.class);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ tce.setAttemptId(attemptId);
+ tce.setStatus(TaskAttemptCompletionEventStatus.SUCCEEDED);
+ job.handle(new JobTaskAttemptCompletedEvent(tce));
+ // complete the task itself
+ job.handle(new JobTaskEvent(taskId, TaskState.SUCCEEDED));
+ Assert.assertEquals(JobState.RUNNING, job.getState());
+ }
+ }
+
+ // add an event for a node transition
+ NodeReport firstMapperNodeReport = nodeReports.get(0);
+ NodeReport secondMapperNodeReport = nodeReports.get(1);
+ job.handle(new JobUpdatedNodesEvent(job.getID(),
+ Collections.singletonList(firstMapperNodeReport)));
+ // complete the reducer
+ for (TaskId taskId: job.tasks.keySet()) {
+ if (taskId.getTaskType() == TaskType.REDUCE) {
+ job.handle(new JobTaskEvent(taskId, TaskState.SUCCEEDED));
+ }
+ }
+ // add another event for a node transition for the other mapper
+ // this should not trigger rescheduling
+ job.handle(new JobUpdatedNodesEvent(job.getID(),
+ Collections.singletonList(secondMapperNodeReport)));
+ // complete the first mapper that was rescheduled
+ TaskId firstMapper = nodeReportsToTaskIds.get(firstMapperNodeReport);
+ job.handle(new JobTaskEvent(firstMapper, TaskState.SUCCEEDED));
+ // verify the state is moving to committing
+ assertJobState(job, JobStateInternal.COMMITTING);
+
+ // let the committer complete and verify the job succeeds
+ syncBarrier.await();
+ assertJobState(job, JobStateInternal.SUCCEEDED);
+
+ dispatcher.stop();
+ commitHandler.stop();
+ }
+
public static void main(String[] args) throws Exception {
TestJobImpl t = new TestJobImpl();
t.testJobNoTasks();