[3/6] mesos git commit: Improved Mesos containerizer logging and documentation.

[an...@apache.org]

Improved Mesos containerizer logging and documentation.

Review: https://reviews.apache.org/r/49651


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/008e0443
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/008e0443
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/008e0443

Branch: refs/heads/0.28.x
Commit: 008e04433026aaec49779197c4a7b6655d5bb693
Parents: 90b5be8
Author: Jiang Yan Xu <xu...@apple.com>
Authored: Fri Jul 1 15:25:54 2016 -0700
Committer: Anand Mazumdar <an...@apache.org>
Committed: Tue Nov 22 20:31:05 2016 -0800

----------------------------------------------------------------------
 src/slave/containerizer/mesos/containerizer.cpp | 21 +++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/008e0443/src/slave/containerizer/mesos/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp
index b1dc7a8..59c2cd8 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -1288,7 +1288,10 @@ Future<containerizer::Termination> MesosContainerizerProcess::wait(
     const ContainerID& containerId)
 {
   if (!containers_.contains(containerId)) {
-    return Failure("Unknown container: " + stringify(containerId));
+    // See the comments in destroy() for race conditions which lead
+    // to "unknown containers".
+    return Failure("Unknown container (could have already been destroyed): " +
+                   stringify(containerId));
   }
 
   return containers_[containerId]->promise.future();
@@ -1451,14 +1454,26 @@ void MesosContainerizerProcess::destroy(
     const ContainerID& containerId)
 {
   if (!containers_.contains(containerId)) {
-    LOG(WARNING) << "Ignoring destroy of unknown container: " << containerId;
+    // This can happen due to the race between destroys initiated by
+    // the launch failure, the terminated executor and the agent so
+    // the same container is destroyed multiple times in reaction to
+    // one failure. e.g., a stuck fetcher results in:
+    // - The agent invoking destroy(), which kills the fetcher and
+    //   the executor.
+    // - The agent invoking destroy() again for the failed launch
+    //   (due to the fetcher getting killed).
+    // - The containerizer invoking destroy() for the reaped executor.
+    //
+    // The guard here and `if (container->state == DESTROYING)` below
+    // make sure redundant destroys short-circuit.
+    VLOG(1) << "Ignoring destroy of unknown container: " << containerId;
     return;
   }
 
   Container* container = containers_[containerId].get();
 
   if (container->state == DESTROYING) {
-    // Destroy has already been initiated.
+    VLOG(1) << "Destroy has already been initiated for '" << containerId << "'";
     return;
   }