You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/08/13 21:50:05 UTC
[21/23] git commit: Fixed slave to properly handle terminated tasks
that have pending updates.
Fixed slave to properly handle terminated tasks that have pending
updates.
Review: https://reviews.apache.org/r/13187
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/93784622
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/93784622
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/93784622
Branch: refs/heads/vinod/0.13.0
Commit: 93784622e0e8557e09feab8dd446594c15b5b3fc
Parents: 5c19b04
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed Jul 31 22:26:49 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Aug 5 11:22:24 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 31 +++++++++++++-------------
src/slave/slave.hpp | 4 ++--
src/slave/status_update_manager.cpp | 12 +++++-----
src/slave/status_update_manager.hpp | 26 ++++++++++-----------
src/tests/slave_recovery_tests.cpp | 3 +--
src/tests/status_update_manager_tests.cpp | 6 -----
6 files changed, 38 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c56dafc..c40a9ac 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -850,7 +850,7 @@ void Slave::_runTask(
<< " because the framework is terminating";
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
return;
}
@@ -875,7 +875,7 @@ void Slave::_runTask(
statusUpdate(update);
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
return;
@@ -1152,7 +1152,7 @@ void Slave::shutdownFramework(const FrameworkID& frameworkId)
// NOTE: We call remove here to ensure we can remove an
// executor (of a terminating framework) that is terminated
// but waiting for acknowledgements.
- remove(framework, executor);
+ removeExecutor(framework, executor);
} else {
// Executor is terminating. Ignore.
}
@@ -1331,11 +1331,6 @@ void Slave::_statusUpdateAcknowledgement(
return;
}
- if (!future.get().get()) {
- // The status update manager ignored the acknowledgement (e.g., duplicate).
- return;
- }
-
VLOG(1) << "Status update manager successfully handled status update"
<< " acknowledgement " << uuid
<< " for task " << taskId
@@ -1372,14 +1367,16 @@ void Slave::_statusUpdateAcknowledgement(
executor->state == Executor::TERMINATED)
<< executor->state;
- if (executor->terminatedTasks.contains(taskId)) {
+ // If the task has reached terminal state and all its updates have
+ // been acknowledged, mark it completed.
+ if (executor->terminatedTasks.contains(taskId) && !future.get().get()) {
executor->completeTask(taskId);
}
// Remove the executor if it has terminated and there are no more
// incomplete tasks.
if (executor->state == Executor::TERMINATED && !executor->incompleteTasks()) {
- remove(framework, executor);
+ removeExecutor(framework, executor);
}
}
@@ -2161,7 +2158,7 @@ void Slave::executorTerminated(
// there are no incomplete tasks.
if (framework->state == Framework::TERMINATING ||
!executor->incompleteTasks()) {
- remove(framework, executor);
+ removeExecutor(framework, executor);
}
break;
}
@@ -2174,7 +2171,7 @@ void Slave::executorTerminated(
}
-void Slave::remove(Framework* framework, Executor* executor)
+void Slave::removeExecutor(Framework* framework, Executor* executor)
{
CHECK_NOTNULL(framework);
CHECK_NOTNULL(executor);
@@ -2234,12 +2231,12 @@ void Slave::remove(Framework* framework, Executor* executor)
// Remove this framework if it has no pending executors and tasks.
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
}
-void Slave::remove(Framework* framework)
+void Slave::removeFramework(Framework* framework)
{
CHECK_NOTNULL(framework);
@@ -2665,7 +2662,7 @@ void Slave::recoverFramework(const FrameworkState& state, bool reconnect)
// Remove the framework in case we didn't recover any executors.
if (framework->executors.empty()) {
- remove(framework);
+ removeFramework(framework);
}
}
@@ -2950,6 +2947,8 @@ void Executor::terminateTask(
const TaskID& taskId,
const mesos::TaskState& state)
{
+ VLOG(1) << "Terminating task " << taskId;
+
Task* task = NULL;
// Remove the task if it's queued.
if (queuedTasks.contains(taskId)) {
@@ -2970,6 +2969,8 @@ void Executor::terminateTask(
void Executor::completeTask(const TaskID& taskId)
{
+ VLOG(1) << "Completing task " << taskId;
+
CHECK(terminatedTasks.contains(taskId))
<< "Failed to find terminated task " << taskId;
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index a9fd9c9..010159c 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -246,10 +246,10 @@ protected:
void recoverFramework(const state::FrameworkState& state, bool reconnect);
// Removes and garbage collects the executor.
- void remove(Framework* framework, Executor* executor);
+ void removeExecutor(Framework* framework, Executor* executor);
// Removes and garbage collects the framework.
- void remove(Framework* framework);
+ void removeFramework(Framework* framework);
private:
Slave(const Slave&); // No copying.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.cpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.cpp b/src/slave/status_update_manager.cpp
index 9e9e4e2..fbc09e6 100644
--- a/src/slave/status_update_manager.cpp
+++ b/src/slave/status_update_manager.cpp
@@ -380,17 +380,17 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
// This might happen if we retried a status update and got back
// acknowledgments for both the original and the retried update.
if (update.isNone()) {
- LOG(WARNING) << "Ignoring unexpected status update acknowledgment " << uuid
- << " for task " << taskId
- << " of framework " << frameworkId;
- return false;
+ return Error(
+ "Unexpected status update acknowledgment " + uuid.toString() +
+ " for task " + stringify(taskId) +
+ " of framework " + stringify(frameworkId));
}
// Handle the acknowledgement.
Try<bool> result =
stream->acknowledgement(taskId, frameworkId, uuid, update.get());
- if (result.isError() || !result.get()) {
+ if (result.isError()) {
return result;
}
@@ -415,7 +415,7 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
stream->timeout = forward(next.get());
}
- return true;
+ return result;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.hpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.hpp b/src/slave/status_update_manager.hpp
index 795e74c..d166e29 100644
--- a/src/slave/status_update_manager.hpp
+++ b/src/slave/status_update_manager.hpp
@@ -36,6 +36,7 @@
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/protobuf.hpp>
+#include <stout/stringify.hpp>
#include <stout/utils.hpp>
#include <stout/uuid.hpp>
@@ -102,9 +103,10 @@ public:
// Checkpoints the status update to disk if necessary.
// Also, sends the next pending status update, if any.
- // @return True if the ACK is handled successfully (e.g., checkpointed).
- // False if the ACK was ignored because it was a duplicate.
- // Error if there are any errors (e.g., checkpointing).
+ // @return true if the ACK is handled successfully (e.g., checkpointed)
+ // and the task's status update stream is not terminated.
+ // false same as above except the status update stream is terminated.
+ // Error if there are any errors (e.g., duplicate, checkpointing).
process::Future<Try<bool> > acknowledgement(
const TaskID& taskId,
const FrameworkID& frameworkId,
@@ -236,20 +238,18 @@ struct StatusUpdateStream
}
if (acknowledged.contains(uuid)) {
- LOG(WARNING) << "Ignoring duplicate status update acknowledgment " << uuid
- << " for update " << update;
- return false;
+ return Error("Duplicate status update acknowledgment " + uuid.toString() +
+ " for update " + stringify(update));
}
// This might happen if we retried a status update and got back
// acknowledgments for both the original and the retried update.
if (uuid != UUID::fromBytes(update.uuid())) {
- LOG(WARNING)
- << "Ignoring unexpected status update acknowledgement "
- << "(received " << uuid
- << ", expecting " << UUID::fromBytes(update.uuid())
- << ") for update " << update;
- return false;
+ return Error(
+ "Unexpected status update acknowledgement (received " +
+ uuid.toString() +
+ ", expecting " + UUID::fromBytes(update.uuid()).toString() +
+ ") for update " + stringify(update));
}
// Handle the ACK, checkpointing if necessary.
@@ -258,7 +258,7 @@ struct StatusUpdateStream
return Error(result.error());
}
- return true;
+ return !terminated;
}
// Returns the next update (or none, if empty) in the queue.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index daae66e..02425f6 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -902,8 +902,7 @@ TYPED_TEST(SlaveRecoveryTest, NonCheckpointingFramework)
Clock::pause();
- Future<Nothing> updateFramework =
- FUTURE_DISPATCH(_, &Slave::updateFramework);
+ Future<Nothing> updateFramework = FUTURE_DISPATCH(_, &Slave::updateFramework);
// Simulate a 'UpdateFrameworkMessage' to ensure framework pid is
// not being checkpointed.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/status_update_manager_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/status_update_manager_tests.cpp b/src/tests/status_update_manager_tests.cpp
index 4239532..6473478 100644
--- a/src/tests/status_update_manager_tests.cpp
+++ b/src/tests/status_update_manager_tests.cpp
@@ -357,9 +357,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreDuplicateStatusUpdateAck)
update.status().task_id(),
update.uuid());
- // TODO(vinod): It would've been great to introspect the first
- // argument of '_statusUpdateAcknowledement()' and ensure that
- // it is 'false'.
AWAIT_READY(duplicateAck);
Clock::resume();
@@ -451,9 +448,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreUnexpectedStatusUpdateAck)
update.status().task_id(),
UUID::random().toBytes());
- // TODO(vinod): It would've been great to introspect the first
- // argument of '_statusUpdateAcknowledement()' and ensure that
- // it is 'false'.
AWAIT_READY(unexpectedAck);
EXPECT_CALL(exec, shutdown(_))