You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2019/11/28 09:04:01 UTC
[hadoop] branch branch-3.2 updated: MAPREDUCE-7249. Fix Invalid
event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job
failure.
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new e830442 MAPREDUCE-7249. Fix Invalid event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
e830442 is described below
commit e8304426b11c29f8b982b55013498427da14c3f4
Author: prabhujoseph <pj...@cloudera.com>
AuthorDate: Thu Nov 28 14:32:26 2019 +0530
MAPREDUCE-7249. Fix Invalid event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
Contributed by Wilfred Spiegelenburg.
---
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 16 +++++++----
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 32 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 998eb03..7fc35c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -475,12 +475,16 @@ public abstract class TaskAttemptImpl implements
TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptEventType.TA_COMMIT_PENDING)
- // Transitions from SUCCESS_CONTAINER_CLEANUP state
- // kill and cleanup the container
- .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
- TaskAttemptStateInternal.SUCCEEDED,
- TaskAttemptEventType.TA_CONTAINER_CLEANED)
- .addTransition(
+ // Transitions from SUCCESS_CONTAINER_CLEANUP state
+ // kill and cleanup the container
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.SUCCEEDED,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED)
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.FAILED,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+ new TooManyFetchFailureTransition())
+ .addTransition(
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 74120b2..7f54ea1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1775,6 +1775,38 @@ public class TestTaskAttempt{
createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
}
+ @Test
+ public void testTooManyFetchFailureWhileContainerCleanup() {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(taImpl.getID().getTaskId()
+ .getJobId(), 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+
+ // move in two steps to the desired state (cannot get there directly)
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_TIMED_OUT));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_CONTAINER_CLEANUP",
+ TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(taImpl.getID(),
+ reduceTAId, "Host"));
+ assertEquals("Task attempt is not in FAILED state",
+ TaskAttemptState.FAILED,
+ taImpl.getState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
private void initResourceTypes() {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org