You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2019/05/10 10:00:00 UTC
[tez] branch master updated (7d5a183 -> 4125639)
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git.
from 7d5a183 TEZ-4062. Speculative attempt scheduling should be aborted when Task has completed
new db5f462 Revert "TEZ-4062. Speculative attempt scheduling should be aborted when Task has completed"
new 4125639 TEZ-4068. Prevent new speculative attempt after task has issued canCommit to an attempt
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[tez] 02/02: TEZ-4068. Prevent new speculative attempt after task
has issued canCommit to an attempt
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
commit 4125639619d5e3252fde62579d431ce5071414ea
Author: Ying Han <hy...@outlook.com>
AuthorDate: Fri May 10 04:59:02 2019 -0500
TEZ-4068. Prevent new speculative attempt after task has issued canCommit to an attempt
Signed-off-by: Jonathan Eagles <je...@apache.org>
---
.../java/org/apache/tez/dag/app/dag/impl/TaskImpl.java | 5 +++++
.../org/apache/tez/dag/app/dag/impl/TestTaskImpl.java | 17 +++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index e563fe9..2d0688f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -1043,6 +1043,11 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
LOG.info("Ignore speculation scheduling since there is no running attempt on task {}.", task.getTaskId());
return;
}
+ if (task.commitAttempt != null) {
+ LOG.info("Ignore speculation scheduling for task {} since commit has started with commitAttempt {}.",
+ task.getTaskId(), task.commitAttempt);
+ return;
+ }
task.addAndScheduleAttempt(earliestUnfinishedAttempt.getID());
}
}
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 2d4adcc..a3de936 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -999,6 +999,23 @@ public class TestTaskImpl {
assertEquals(1, mockTask.getAttemptList().size());
}
+ @Test(timeout = 20000)
+ public void testIgnoreSpeculationAfterOriginalAttemptCommit() {
+ TezTaskID taskId = getNewTaskID();
+ scheduleTaskAttempt(taskId);
+ MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
+ launchTaskAttempt(firstAttempt.getID());
+ updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);
+ // Mock commit of the first task attempt
+ mockTask.canCommit(firstAttempt.getID());
+
+ // Verify the speculation scheduling is ignored and no speculative attempt was added to the task
+ mockTask.handle(createTaskTAAddSpecAttempt(firstAttempt.getID()));
+ MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
+ launchTaskAttempt(specAttempt.getID());
+ assertEquals(1, mockTask.getAttemptList().size());
+ }
+
@SuppressWarnings("rawtypes")
@Test
public void testSucceededAttemptStatusWithRetroActiveFailures() throws InterruptedException {
[tez] 01/02: Revert "TEZ-4062. Speculative attempt scheduling
should be aborted when Task has completed"
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
commit db5f462f9a6ac7faf5fde686bab401afb55ab66b
Author: Jonathan Eagles <je...@apache.org>
AuthorDate: Fri May 10 04:58:13 2019 -0500
Revert "TEZ-4062. Speculative attempt scheduling should be aborted when Task has completed"
This reverts commit 7d5a1830a51df64b23920901ec2e723999a9dfac.
---
.../java/org/apache/tez/dag/app/dag/impl/TaskImpl.java | 5 -----
.../org/apache/tez/dag/app/dag/impl/TestTaskImpl.java | 17 -----------------
2 files changed, 22 deletions(-)
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 2d0688f..e563fe9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -1043,11 +1043,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
LOG.info("Ignore speculation scheduling since there is no running attempt on task {}.", task.getTaskId());
return;
}
- if (task.commitAttempt != null) {
- LOG.info("Ignore speculation scheduling for task {} since commit has started with commitAttempt {}.",
- task.getTaskId(), task.commitAttempt);
- return;
- }
task.addAndScheduleAttempt(earliestUnfinishedAttempt.getID());
}
}
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index a3de936..2d4adcc 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -999,23 +999,6 @@ public class TestTaskImpl {
assertEquals(1, mockTask.getAttemptList().size());
}
- @Test(timeout = 20000)
- public void testIgnoreSpeculationAfterOriginalAttemptCommit() {
- TezTaskID taskId = getNewTaskID();
- scheduleTaskAttempt(taskId);
- MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
- launchTaskAttempt(firstAttempt.getID());
- updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);
- // Mock commit of the first task attempt
- mockTask.canCommit(firstAttempt.getID());
-
- // Verify the speculation scheduling is ignored and no speculative attempt was added to the task
- mockTask.handle(createTaskTAAddSpecAttempt(firstAttempt.getID()));
- MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
- launchTaskAttempt(specAttempt.getID());
- assertEquals(1, mockTask.getAttemptList().size());
- }
-
@SuppressWarnings("rawtypes")
@Test
public void testSucceededAttemptStatusWithRetroActiveFailures() throws InterruptedException {