You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/08/13 21:50:05 UTC

[21/23] git commit: Fixed slave to properly handle terminated tasks that have pending updates.

Fixed slave to properly handle terminated tasks that have pending
updates.

Review: https://reviews.apache.org/r/13187


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/93784622
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/93784622
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/93784622

Branch: refs/heads/vinod/0.13.0
Commit: 93784622e0e8557e09feab8dd446594c15b5b3fc
Parents: 5c19b04
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed Jul 31 22:26:49 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Aug 5 11:22:24 2013 -0700

----------------------------------------------------------------------
 src/slave/slave.cpp                       | 31 +++++++++++++-------------
 src/slave/slave.hpp                       |  4 ++--
 src/slave/status_update_manager.cpp       | 12 +++++-----
 src/slave/status_update_manager.hpp       | 26 ++++++++++-----------
 src/tests/slave_recovery_tests.cpp        |  3 +--
 src/tests/status_update_manager_tests.cpp |  6 -----
 6 files changed, 38 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c56dafc..c40a9ac 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -850,7 +850,7 @@ void Slave::_runTask(
                  << " because the framework is terminating";
 
     if (framework->executors.empty() && framework->pending.empty()) {
-      remove(framework);
+      removeFramework(framework);
     }
     return;
   }
@@ -875,7 +875,7 @@ void Slave::_runTask(
     statusUpdate(update);
 
     if (framework->executors.empty() && framework->pending.empty()) {
-      remove(framework);
+      removeFramework(framework);
     }
 
     return;
@@ -1152,7 +1152,7 @@ void Slave::shutdownFramework(const FrameworkID& frameworkId)
           // NOTE: We call remove here to ensure we can remove an
           // executor (of a terminating framework) that is terminated
           // but waiting for acknowledgements.
-          remove(framework, executor);
+          removeExecutor(framework, executor);
         } else {
           // Executor is terminating. Ignore.
         }
@@ -1331,11 +1331,6 @@ void Slave::_statusUpdateAcknowledgement(
     return;
   }
 
-  if (!future.get().get()) {
-    // The status update manager ignored the acknowledgement (e.g., duplicate).
-    return;
-  }
-
   VLOG(1) << "Status update manager successfully handled status update"
           << " acknowledgement " << uuid
           << " for task " << taskId
@@ -1372,14 +1367,16 @@ void Slave::_statusUpdateAcknowledgement(
         executor->state == Executor::TERMINATED)
     << executor->state;
 
-  if (executor->terminatedTasks.contains(taskId)) {
+  // If the task has reached terminal state and all its updates have
+  // been acknowledged, mark it completed.
+  if (executor->terminatedTasks.contains(taskId) && !future.get().get()) {
     executor->completeTask(taskId);
   }
 
   // Remove the executor if it has terminated and there are no more
   // incomplete tasks.
   if (executor->state == Executor::TERMINATED && !executor->incompleteTasks()) {
-    remove(framework, executor);
+    removeExecutor(framework, executor);
   }
 }
 
@@ -2161,7 +2158,7 @@ void Slave::executorTerminated(
       // there are no incomplete tasks.
       if (framework->state == Framework::TERMINATING ||
           !executor->incompleteTasks()) {
-        remove(framework, executor);
+        removeExecutor(framework, executor);
       }
       break;
     }
@@ -2174,7 +2171,7 @@ void Slave::executorTerminated(
 }
 
 
-void Slave::remove(Framework* framework, Executor* executor)
+void Slave::removeExecutor(Framework* framework, Executor* executor)
 {
   CHECK_NOTNULL(framework);
   CHECK_NOTNULL(executor);
@@ -2234,12 +2231,12 @@ void Slave::remove(Framework* framework, Executor* executor)
 
   // Remove this framework if it has no pending executors and tasks.
   if (framework->executors.empty() && framework->pending.empty()) {
-    remove(framework);
+    removeFramework(framework);
   }
 }
 
 
-void Slave::remove(Framework* framework)
+void Slave::removeFramework(Framework* framework)
 {
   CHECK_NOTNULL(framework);
 
@@ -2665,7 +2662,7 @@ void Slave::recoverFramework(const FrameworkState& state, bool reconnect)
 
   // Remove the framework in case we didn't recover any executors.
   if (framework->executors.empty()) {
-    remove(framework);
+    removeFramework(framework);
   }
 }
 
@@ -2950,6 +2947,8 @@ void Executor::terminateTask(
     const TaskID& taskId,
     const mesos::TaskState& state)
 {
+  VLOG(1) << "Terminating task " << taskId;
+
   Task* task = NULL;
   // Remove the task if it's queued.
   if (queuedTasks.contains(taskId)) {
@@ -2970,6 +2969,8 @@ void Executor::terminateTask(
 
 void Executor::completeTask(const TaskID& taskId)
 {
+  VLOG(1) << "Completing task " << taskId;
+
   CHECK(terminatedTasks.contains(taskId))
     << "Failed to find terminated task " << taskId;
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index a9fd9c9..010159c 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -246,10 +246,10 @@ protected:
   void recoverFramework(const state::FrameworkState& state, bool reconnect);
 
   // Removes and garbage collects the executor.
-  void remove(Framework* framework, Executor* executor);
+  void removeExecutor(Framework* framework, Executor* executor);
 
   // Removes and garbage collects the framework.
-  void remove(Framework* framework);
+  void removeFramework(Framework* framework);
 
 private:
   Slave(const Slave&);              // No copying.

http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.cpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.cpp b/src/slave/status_update_manager.cpp
index 9e9e4e2..fbc09e6 100644
--- a/src/slave/status_update_manager.cpp
+++ b/src/slave/status_update_manager.cpp
@@ -380,17 +380,17 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
   // This might happen if we retried a status update and got back
   // acknowledgments for both the original and the retried update.
   if (update.isNone()) {
-    LOG(WARNING) << "Ignoring unexpected status update acknowledgment " << uuid
-                 << " for task " << taskId
-                 << " of framework " << frameworkId;
-    return false;
+    return Error(
+        "Unexpected status update acknowledgment " + uuid.toString() +
+        " for task " + stringify(taskId) +
+        " of framework " + stringify(frameworkId));
   }
 
   // Handle the acknowledgement.
   Try<bool> result =
     stream->acknowledgement(taskId, frameworkId, uuid, update.get());
 
-  if (result.isError() || !result.get()) {
+  if (result.isError()) {
     return result;
   }
 
@@ -415,7 +415,7 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
     stream->timeout = forward(next.get());
   }
 
-  return true;
+  return result;
 }
 
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.hpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.hpp b/src/slave/status_update_manager.hpp
index 795e74c..d166e29 100644
--- a/src/slave/status_update_manager.hpp
+++ b/src/slave/status_update_manager.hpp
@@ -36,6 +36,7 @@
 #include <stout/option.hpp>
 #include <stout/os.hpp>
 #include <stout/protobuf.hpp>
+#include <stout/stringify.hpp>
 #include <stout/utils.hpp>
 #include <stout/uuid.hpp>
 
@@ -102,9 +103,10 @@ public:
 
   // Checkpoints the status update to disk if necessary.
   // Also, sends the next pending status update, if any.
-  // @return True if the ACK is handled successfully (e.g., checkpointed).
-  //         False if the ACK was ignored because it was a duplicate.
-  //         Error if there are any errors (e.g., checkpointing).
+  // @return true if the ACK is handled successfully (e.g., checkpointed)
+  //              and the task's status update stream is not terminated.
+  //         false same as above except the status update stream is terminated.
+  //         Error if there are any errors (e.g., duplicate, checkpointing).
   process::Future<Try<bool> > acknowledgement(
       const TaskID& taskId,
       const FrameworkID& frameworkId,
@@ -236,20 +238,18 @@ struct StatusUpdateStream
     }
 
     if (acknowledged.contains(uuid)) {
-      LOG(WARNING) << "Ignoring duplicate status update acknowledgment " << uuid
-                   << " for update " << update;
-      return false;
+      return Error("Duplicate status update acknowledgment " + uuid.toString() +
+                   " for update " + stringify(update));
     }
 
     // This might happen if we retried a status update and got back
     // acknowledgments for both the original and the retried update.
     if (uuid != UUID::fromBytes(update.uuid())) {
-      LOG(WARNING)
-        << "Ignoring unexpected status update acknowledgement "
-        << "(received " << uuid
-        << ", expecting " << UUID::fromBytes(update.uuid())
-        << ") for update " << update;
-      return false;
+      return Error(
+        "Unexpected status update acknowledgement (received " +
+        uuid.toString() +
+        ", expecting " + UUID::fromBytes(update.uuid()).toString() +
+        ") for update " + stringify(update));
     }
 
     // Handle the ACK, checkpointing if necessary.
@@ -258,7 +258,7 @@ struct StatusUpdateStream
       return Error(result.error());
     }
 
-    return true;
+    return !terminated;
   }
 
   // Returns the next update (or none, if empty) in the queue.

http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index daae66e..02425f6 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -902,8 +902,7 @@ TYPED_TEST(SlaveRecoveryTest, NonCheckpointingFramework)
 
   Clock::pause();
 
-  Future<Nothing> updateFramework =
-    FUTURE_DISPATCH(_, &Slave::updateFramework);
+  Future<Nothing> updateFramework = FUTURE_DISPATCH(_, &Slave::updateFramework);
 
   // Simulate a 'UpdateFrameworkMessage' to ensure framework pid is
   // not being checkpointed.

http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/status_update_manager_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/status_update_manager_tests.cpp b/src/tests/status_update_manager_tests.cpp
index 4239532..6473478 100644
--- a/src/tests/status_update_manager_tests.cpp
+++ b/src/tests/status_update_manager_tests.cpp
@@ -357,9 +357,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreDuplicateStatusUpdateAck)
       update.status().task_id(),
       update.uuid());
 
-  // TODO(vinod): It would've been great to introspect the first
-  // argument of '_statusUpdateAcknowledement()' and ensure that
-  // it is 'false'.
   AWAIT_READY(duplicateAck);
 
   Clock::resume();
@@ -451,9 +448,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreUnexpectedStatusUpdateAck)
       update.status().task_id(),
       UUID::random().toBytes());
 
-  // TODO(vinod): It would've been great to introspect the first
-  // argument of '_statusUpdateAcknowledement()' and ensure that
-  // it is 'false'.
   AWAIT_READY(unexpectedAck);
 
   EXPECT_CALL(exec, shutdown(_))