You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/08/17 21:43:36 UTC
[hadoop] 01/02: MAPREDUCE-7240. Fix Invalid event:
TA_TOO_MANY_FETCH_FAILURE at SUCCESS_FINISHING_CONTAINER.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f4f00d56a14f0473b345ad0c1bebb8c0b2932bc5
Author: prabhujoseph <pj...@cloudera.com>
AuthorDate: Wed Nov 27 16:51:45 2019 +0530
MAPREDUCE-7240. Fix Invalid event: TA_TOO_MANY_FETCH_FAILURE at SUCCESS_FINISHING_CONTAINER.
Contributed by Huachao and Peter Bacsko. Reviewed by Wilfred Spiegelenburg.
(cherry picked from commit 6a4966f336c7b86a69d292b6157315e25963e527)
---
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 8 +++++++
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 25 ++++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3020b14..7dcdff7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -376,6 +376,10 @@ public abstract class TaskAttemptImpl implements
TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+ .addTransition(TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ TaskAttemptStateInternal.FAILED,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+ new TooManyFetchFailureTransition())
// ignore-able events
.addTransition(TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
@@ -2144,6 +2148,10 @@ public abstract class TaskAttemptImpl implements
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+ if (taskAttempt.getInternalState() ==
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER) {
+ sendContainerCleanup(taskAttempt, event);
+ }
TaskAttemptTooManyFetchFailureEvent fetchFailureEvent =
(TaskAttemptTooManyFetchFailureEvent) event;
// too many fetch failure can only happen for map tasks
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index e055798..9fab43f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1780,6 +1780,31 @@ public class TestTaskAttempt{
ResourceUtils.resetResourceTypes(conf);
}
+ @Test
+ public void testTooManyFetchFailureWhileSuccessFinishing() throws Exception {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(taImpl.getID().getTaskId()
+ .getJobId(), 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(taImpl.getID(),
+ reduceTAId, "Host"));
+ assertEquals("Task attempt is not in FAILED state",
+ TaskAttemptState.FAILED,
+ taImpl.getState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
private void setupTaskAttemptFinishingMonitor(
EventHandler eventHandler, JobConf jobConf, AppContext appCtx) {
TaskAttemptFinishingMonitor taskAttemptFinishingMonitor =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org