You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/08/17 21:43:35 UTC
[hadoop] branch branch-2.10 updated (283def8 -> 30c95dd)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a change to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 283def8 MAPREDUCE-7069. Add ability to specify user environment variables individually. Contributed by Jim Brennan
new f4f00d5 MAPREDUCE-7240. Fix Invalid event: TA_TOO_MANY_FETCH_FAILURE at SUCCESS_FINISHING_CONTAINER.
new 30c95dd MAPREDUCE-7249. Fix Invalid event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 24 ++++++---
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 57 ++++++++++++++++++++++
2 files changed, 75 insertions(+), 6 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: MAPREDUCE-7249. Fix Invalid event
TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 30c95ddaff2619d00bd232e52086ea57ed3664a1
Author: prabhujoseph <pj...@cloudera.com>
AuthorDate: Thu Nov 28 14:34:50 2019 +0530
MAPREDUCE-7249. Fix Invalid event TA_TOO_MANY_FETCH_FAILURE at SUCCESS_CONTAINER_CLEANUP causes job failure.
Contributed by Wilfred Spiegelenburg.
(cherry picked from commit a97f7776bd05f957a998b4883141855cad5afb31)
---
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 16 +++++++----
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 32 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 7dcdff7..d4947d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -469,12 +469,16 @@ public abstract class TaskAttemptImpl implements
TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptEventType.TA_COMMIT_PENDING)
- // Transitions from SUCCESS_CONTAINER_CLEANUP state
- // kill and cleanup the container
- .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
- TaskAttemptStateInternal.SUCCEEDED,
- TaskAttemptEventType.TA_CONTAINER_CLEANED)
- .addTransition(
+ // Transitions from SUCCESS_CONTAINER_CLEANUP state
+ // kill and cleanup the container
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.SUCCEEDED,
+ TaskAttemptEventType.TA_CONTAINER_CLEANED)
+ .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ TaskAttemptStateInternal.FAILED,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+ new TooManyFetchFailureTransition())
+ .addTransition(
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 9fab43f..4c73955 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1773,6 +1773,38 @@ public class TestTaskAttempt{
createReduceTaskAttemptImplForTest(eventHandler, clock, jobConf);
}
+ @Test
+ public void testTooManyFetchFailureWhileContainerCleanup() {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(taImpl.getID().getTaskId()
+ .getJobId(), 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+
+ // move in two steps to the desired state (cannot get there directly)
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_TIMED_OUT));
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_CONTAINER_CLEANUP",
+ TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(taImpl.getID(),
+ reduceTAId, "Host"));
+ assertEquals("Task attempt is not in FAILED state",
+ TaskAttemptState.FAILED,
+ taImpl.getState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
private void initResourceTypes() {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: MAPREDUCE-7240. Fix Invalid event:
TA_TOO_MANY_FETCH_FAILURE at SUCCESS_FINISHING_CONTAINER.
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f4f00d56a14f0473b345ad0c1bebb8c0b2932bc5
Author: prabhujoseph <pj...@cloudera.com>
AuthorDate: Wed Nov 27 16:51:45 2019 +0530
MAPREDUCE-7240. Fix Invalid event: TA_TOO_MANY_FETCH_FAILURE at SUCCESS_FINISHING_CONTAINER.
Contributed by Huachao and Peter Bacsko. Reviewed by Wilfred Spiegelenburg.
(cherry picked from commit 6a4966f336c7b86a69d292b6157315e25963e527)
---
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 8 +++++++
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 25 ++++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3020b14..7dcdff7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -376,6 +376,10 @@ public abstract class TaskAttemptImpl implements
TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
+ .addTransition(TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ TaskAttemptStateInternal.FAILED,
+ TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
+ new TooManyFetchFailureTransition())
// ignore-able events
.addTransition(TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
@@ -2144,6 +2148,10 @@ public abstract class TaskAttemptImpl implements
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+ if (taskAttempt.getInternalState() ==
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER) {
+ sendContainerCleanup(taskAttempt, event);
+ }
TaskAttemptTooManyFetchFailureEvent fetchFailureEvent =
(TaskAttemptTooManyFetchFailureEvent) event;
// too many fetch failure can only happen for map tasks
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index e055798..9fab43f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -1780,6 +1780,31 @@ public class TestTaskAttempt{
ResourceUtils.resetResourceTypes(conf);
}
+ @Test
+ public void testTooManyFetchFailureWhileSuccessFinishing() throws Exception {
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(taImpl.getID().getTaskId()
+ .getJobId(), 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+
+ taImpl.handle(new TaskAttemptEvent(taImpl.getID(),
+ TaskAttemptEventType.TA_DONE));
+
+ assertEquals("Task attempt's internal state is not " +
+ "SUCCESS_FINISHING_CONTAINER",
+ TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER,
+ taImpl.getInternalState());
+
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(taImpl.getID(),
+ reduceTAId, "Host"));
+ assertEquals("Task attempt is not in FAILED state",
+ TaskAttemptState.FAILED,
+ taImpl.getState());
+ assertFalse("InternalError occurred", eventHandler.internalError);
+ }
+
private void setupTaskAttemptFinishingMonitor(
EventHandler eventHandler, JobConf jobConf, AppContext appCtx) {
TaskAttemptFinishingMonitor taskAttemptFinishingMonitor =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org