You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/08/17 21:43:37 UTC
[hadoop] 02/02: MAPREDUCE-7249. Fix Invalid event
TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 30c95ddaff2619d00bd232e52086ea57ed3664a1
Author: prabhujoseph <pj...@cloudera.com>
AuthorDate: Thu Nov 28 14:34:50 2019 +0530
MAPREDUCE-7249. Fix Invalid event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
Contributed by Wilfred Spiegelenburg.
(cherry picked from commit a97f7776bd05f957a998b4883141855cad5afb31)
---
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 16 +++++++----
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 32 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 7dcdff7..d4947d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -469,12 +469,16 @@ public abstract class TaskAttemptImpl implements
TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptEventType.TA_COMMIT_PENDING)
- // Transitions from SUCCESS_CONTAINER_CLEANUP state
- // kill and cleanup the container
- .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
- TaskAttemptStateInternal.SUCCEEDED,
- TaskAttemptEventType.TA_CONTAINER_CLEANED)
- .addTransition(
+ // Transitions from SUCCESS_CONTAINER_CLEANUP state
+ // kill and cleanup the container
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.SUCCEEDED,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED)
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.FAILED,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+ new TooManyFetchFailureTransition())
+ .addTransition(
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 9fab43f..4c73955 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1773,6 +1773,38 @@ public class TestTaskAttempt{
createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
}
+ @Test
+ public void testTooManyFetchFailureWhileContainerCleanup() {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(taImpl.getID().getTaskId()
+ .getJobId(), 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+
+ // move in two steps to the desired state (cannot get there directly)
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_TIMED_OUT));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_CONTAINER_CLEANUP",
+ TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(taImpl.getID(),
+ reduceTAId, "Host"));
+ assertEquals("Task attempt is not in FAILED state",
+ TaskAttemptState.FAILED,
+ taImpl.getState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
private void initResourceTypes() {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org