You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2017/01/10 21:33:27 UTC

mesos git commit: Fixed a bug in the default executor around not committing suicide.

Repository: mesos
Updated Branches:
  refs/heads/master 798fa681c -> 3efcd33f4


Fixed a bug in the default executor around not committing suicide.

This bug is only observed when the task group contains a single task.
The default executor was not committing suicide when this single task
used to exit with a non-zero status code as per the default restart
policy.

Review: https://reviews.apache.org/r/55157/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3efcd33f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3efcd33f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3efcd33f

Branch: refs/heads/master
Commit: 3efcd33f440c7e56c137bfb7cd953ee35e4b3aa5
Parents: 798fa68
Author: Anand Mazumdar <an...@apache.org>
Authored: Tue Jan 10 13:08:03 2017 -0800
Committer: Anand Mazumdar <an...@apache.org>
Committed: Tue Jan 10 13:08:03 2017 -0800

----------------------------------------------------------------------
 src/launcher/default_executor.cpp    |  34 +++++---
 src/tests/default_executor_tests.cpp | 140 ++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/3efcd33f/src/launcher/default_executor.cpp
----------------------------------------------------------------------
diff --git a/src/launcher/default_executor.cpp b/src/launcher/default_executor.cpp
index fd99639..57e4799 100644
--- a/src/launcher/default_executor.cpp
+++ b/src/launcher/default_executor.cpp
@@ -653,28 +653,34 @@ protected:
       update(taskId, taskState, message, None());
     }
 
-    LOG(INFO) << "Successfully waited for child container " << containerId
-              << " of task '" << taskId << "'"
-              << " in state " << stringify(taskState);
-
     CHECK(containers.contains(containerId));
     containers.erase(containerId);
 
-    // The default restart policy for a task group is to kill all the
-    // remaining child containers if one of them terminated with a
-    // non-zero exit code. Ignore if a shutdown is in progress.
-    if (!shuttingDown && taskState == TASK_FAILED) {
-      LOG(ERROR)
-        << "Child container " << containerId << " terminated with status "
-        << (status.isSome() ? WSTRINGIFY(status.get()) : "unknown");
-      shutdown();
-      return;
-    }
+    LOG(INFO)
+      << "Child container " << containerId << " of task '" << taskId
+      << "' in state " << stringify(taskState) << " terminated with status "
+      << (status.isSome() ? WSTRINGIFY(status.get()) : "unknown");
 
     // Shutdown the executor if all the active child containers have terminated.
     if (containers.empty()) {
       __shutdown();
     }
+
+    // Ignore if the executor is already in the process of shutting down.
+    if (shuttingDown) {
+      return;
+    }
+
+    // The default restart policy for a task group is to kill all the
+    // remaining child containers if one of them terminated with a
+    // non-zero exit code.
+    if (taskState == TASK_FAILED) {
+      // Kill all the other active containers.
+      //
+      // TODO(anand): Invoke `kill()` once per active container
+      // instead of directly invoking `shutdown()`.
+      shutdown();
+    }
   }
 
   void shutdown()

http://git-wip-us.apache.org/repos/asf/mesos/blob/3efcd33f/src/tests/default_executor_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/default_executor_tests.cpp b/src/tests/default_executor_tests.cpp
index 340e8c8..ec3e854 100644
--- a/src/tests/default_executor_tests.cpp
+++ b/src/tests/default_executor_tests.cpp
@@ -856,6 +856,146 @@ TEST_P(DefaultExecutorTest, ROOT_ContainerStatusForTask)
             status2.container_id().parent());
 }
 
+
+// This test verifies that the default executor commits suicide when the only
+// task in the task group exits with a non-zero status code.
+TEST_P_TEMP_DISABLED_ON_WINDOWS(DefaultExecutorTest, CommitSuicideOnTaskFailure)
+{
+  Try<Owned<cluster::Master>> master = StartMaster();
+  ASSERT_SOME(master);
+
+  auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
+
+  Resources resources =
+    Resources::parse("cpus:0.1;mem:32;disk:32").get();
+
+  FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+
+  ExecutorInfo executorInfo;
+  executorInfo.set_type(ExecutorInfo::DEFAULT);
+
+  executorInfo.mutable_executor_id()->CopyFrom(DEFAULT_EXECUTOR_ID);
+  executorInfo.mutable_resources()->CopyFrom(resources);
+
+  // Disable AuthN on the agent.
+  slave::Flags flags = CreateSlaveFlags();
+  flags.authenticate_http_readwrite = false;
+
+  Owned<MasterDetector> detector = master.get()->createDetector();
+  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
+  ASSERT_SOME(slave);
+
+  Future<Nothing> connected;
+  EXPECT_CALL(*scheduler, connected(_))
+    .WillOnce(FutureSatisfy(&connected));
+
+  v1::scheduler::TestMesos mesos(
+      master.get()->pid,
+      ContentType::PROTOBUF,
+      scheduler);
+
+  AWAIT_READY(connected);
+
+  Future<v1::scheduler::Event::Subscribed> subscribed;
+  EXPECT_CALL(*scheduler, subscribed(_, _))
+    .WillOnce(FutureArg<1>(&subscribed));
+
+  Future<v1::scheduler::Event::Offers> offers;
+  EXPECT_CALL(*scheduler, offers(_, _))
+    .WillOnce(FutureArg<1>(&offers))
+    .WillRepeatedly(Return());
+
+  EXPECT_CALL(*scheduler, heartbeat(_))
+    .WillRepeatedly(Return()); // Ignore heartbeats.
+
+  {
+    Call call;
+    call.set_type(Call::SUBSCRIBE);
+    Call::Subscribe* subscribe = call.mutable_subscribe();
+    subscribe->mutable_framework_info()->CopyFrom(evolve(frameworkInfo));
+
+    mesos.send(call);
+  }
+
+  AWAIT_READY(subscribed);
+
+  v1::FrameworkID frameworkId(subscribed->framework_id());
+
+  // Update `executorInfo` with the subscribed `frameworkId`.
+  executorInfo.mutable_framework_id()->CopyFrom(devolve(frameworkId));
+
+  AWAIT_READY(offers);
+  EXPECT_NE(0, offers->offers().size());
+
+  Future<v1::scheduler::Event::Update> runningUpdate;
+  Future<v1::scheduler::Event::Update> failedUpdate;
+  EXPECT_CALL(*scheduler, update(_, _))
+    .WillOnce(FutureArg<1>(&runningUpdate))
+    .WillOnce(FutureArg<1>(&failedUpdate));
+
+  const v1::Offer& offer = offers->offers(0);
+  const SlaveID slaveId = devolve(offer.agent_id());
+
+  // The task exits with a non-zero status code.
+  v1::TaskInfo taskInfo1 =
+    evolve(createTask(slaveId, resources, "exit 1"));
+
+  v1::TaskGroupInfo taskGroup;
+  taskGroup.add_tasks()->CopyFrom(taskInfo1);
+
+  Future<Nothing> executorFailure;
+  EXPECT_CALL(*scheduler, failure(_, _))
+    .WillOnce(FutureSatisfy(&executorFailure));
+
+  {
+    Call call;
+    call.mutable_framework_id()->CopyFrom(frameworkId);
+    call.set_type(Call::ACCEPT);
+
+    Call::Accept* accept = call.mutable_accept();
+    accept->add_offer_ids()->CopyFrom(offer.id());
+
+    v1::Offer::Operation* operation = accept->add_operations();
+    operation->set_type(v1::Offer::Operation::LAUNCH_GROUP);
+
+    v1::Offer::Operation::LaunchGroup* launchGroup =
+      operation->mutable_launch_group();
+
+    launchGroup->mutable_executor()->CopyFrom(evolve(executorInfo));
+    launchGroup->mutable_task_group()->CopyFrom(taskGroup);
+
+    mesos.send(call);
+  }
+
+  AWAIT_READY(runningUpdate);
+  ASSERT_EQ(TASK_RUNNING, runningUpdate->status().state());
+
+  // Acknowledge the TASK_RUNNING update to receive the next update.
+
+  {
+    Call call;
+    call.mutable_framework_id()->CopyFrom(frameworkId);
+    call.set_type(Call::ACKNOWLEDGE);
+
+    Call::Acknowledge* acknowledge = call.mutable_acknowledge();
+
+    acknowledge->mutable_task_id()->CopyFrom(
+        runningUpdate->status().task_id());
+
+    acknowledge->mutable_agent_id()->CopyFrom(offer.agent_id());
+    acknowledge->set_uuid(runningUpdate->status().uuid());
+
+    mesos.send(call);
+  }
+
+  AWAIT_READY(failedUpdate);
+  ASSERT_EQ(TASK_FAILED, failedUpdate->status().state());
+
+  // The executor should commit suicide when the task exits with
+  // a non-zero status code.
+  AWAIT_READY(executorFailure);
+}
+
 } // namespace tests {
 } // namespace internal {
 } // namespace mesos {