You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/07/05 22:06:15 UTC

[2/4] mesos git commit: Added support for recovered frameworks.

Added support for recovered frameworks.

Previously after a master failover while agents
were reregistering we might not have access
to all FrameworkInfo (which would first be available
once the Framework reregistered). This patch
adds support that once a framework reregisters
we also keep track of recovered FrameworkInfos.

Review: https://reviews.apache.org/r/49607/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3c96ca49
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3c96ca49
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3c96ca49

Branch: refs/heads/master
Commit: 3c96ca497ba2b0a733756f1af5e2dd4b4695d77a
Parents: 90bcedc
Author: Joerg Schad <jo...@mesosphere.io>
Authored: Tue Jul 5 17:05:28 2016 -0500
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Jul 5 17:06:05 2016 -0500

----------------------------------------------------------------------
 src/master/master.cpp       | 54 +++++++++++++++++++++++++++++++++++++---
 src/master/master.hpp       | 10 +++++++-
 src/messages/messages.proto |  1 +
 src/slave/slave.cpp         |  3 ++-
 4 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/3c96ca49/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 78a8889..007f78e 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -919,6 +919,7 @@ void Master::initialize()
       &ReregisterSlaveMessage::checkpointed_resources,
       &ReregisterSlaveMessage::executor_infos,
       &ReregisterSlaveMessage::tasks,
+      &ReregisterSlaveMessage::frameworks,
       &ReregisterSlaveMessage::completed_frameworks,
       &ReregisterSlaveMessage::version);
 
@@ -4753,6 +4754,7 @@ void Master::reregisterSlave(
     const vector<Resource>& checkpointedResources,
     const vector<ExecutorInfo>& executorInfos,
     const vector<Task>& tasks,
+    const vector<FrameworkInfo>& frameworks,
     const vector<Archive::Framework>& completedFrameworks,
     const string& version)
 {
@@ -4770,6 +4772,7 @@ void Master::reregisterSlave(
                      checkpointedResources,
                      executorInfos,
                      tasks,
+                     frameworks,
                      completedFrameworks,
                      version));
     return;
@@ -4888,7 +4891,7 @@ void Master::reregisterSlave(
 
     // Inform the agent of the master's version of its checkpointed
     // resources and the new framework pids for its tasks.
-    __reregisterSlave(slave, tasks);
+    __reregisterSlave(slave, tasks, frameworks);
 
     return;
   }
@@ -4923,6 +4926,7 @@ void Master::reregisterSlave(
                  checkpointedResources,
                  executorInfos,
                  tasks,
+                 frameworks,
                  completedFrameworks,
                  version,
                  lambda::_1));
@@ -4935,6 +4939,7 @@ void Master::_reregisterSlave(
     const vector<Resource>& checkpointedResources,
     const vector<ExecutorInfo>& executorInfos,
     const vector<Task>& tasks,
+    const vector<FrameworkInfo>& frameworks,
     const vector<Archive::Framework>& completedFrameworks,
     const string& version,
     const Future<bool>& readmit)
@@ -4993,18 +4998,26 @@ void Master::_reregisterSlave(
     LOG(INFO) << "Re-registered agent " << *slave
               << " with " << slave->info.resources();
 
-    __reregisterSlave(slave, tasks);
+    __reregisterSlave(slave, tasks, frameworks);
   }
 }
 
 
-void Master::__reregisterSlave(Slave* slave, const vector<Task>& tasks)
+void Master::__reregisterSlave(
+    Slave* slave,
+    const std::vector<Task>& tasks,
+    const vector<FrameworkInfo>& frameworks)
 {
   CHECK_NOTNULL(slave);
 
   // Send the latest framework pids to the slave.
   hashset<FrameworkID> ids;
 
+  // TODO(joerg84): Remove this after a deprecation cycle starting
+  // with the 1.0 release. It is only required if an older
+  // (pre 1.0 agent) reregisters with a newer master.
+  // In that case the agent does not have the 'frameworks' message
+  // set which is used below to retrieve the framework information.
   foreach (const Task& task, tasks) {
     Framework* framework = getFramework(task.framework_id());
 
@@ -5023,6 +5036,31 @@ void Master::__reregisterSlave(Slave* slave, const vector<Task>& tasks)
     }
   }
 
+  foreach (const FrameworkInfo& frameworkInfo, frameworks) {
+    CHECK(frameworkInfo.has_id());
+    Framework* framework = getFramework(frameworkInfo.id());
+
+    if (framework != nullptr && !ids.contains(framework->id())) {
+      UpdateFrameworkMessage message;
+      message.mutable_framework_id()->MergeFrom(framework->id());
+
+      // TODO(anand): We set 'pid' to UPID() for http frameworks
+      // as 'pid' was made optional in 0.24.0. In 0.25.0, we
+      // no longer have to set pid here for http frameworks.
+      message.set_pid(framework->pid.getOrElse(UPID()));
+
+      send(slave->pid, message);
+
+      ids.insert(framework->id());
+    } else {
+      // The framework hasn't yet reregistered with the master,
+      // hence we store the 'FrameworkInfo' in the recovered list.
+      // TODO(joerg84): Consider recovering this information from
+      // registrar instead of from agents.
+      this->frameworks.recovered[frameworkInfo.id()] = frameworkInfo;
+    }
+  }
+
   // NOTE: Here we always send the message. Slaves whose version are
   // less than 0.22.0 will drop it silently which is OK.
   LOG(INFO) << "Sending updated checkpointed resources "
@@ -6179,6 +6217,11 @@ void Master::addFramework(Framework* framework)
 
   frameworks.registered[framework->id()] = framework;
 
+  // Remove from 'frameworks.recovered' if necessary.
+  if (frameworks.recovered.contains(framework->id())) {
+    frameworks.recovered.erase(framework->id());
+  }
+
   if (framework->pid.isSome()) {
     link(framework->pid.get());
   } else {
@@ -6512,6 +6555,11 @@ void Master::removeFramework(Framework* framework)
   frameworks.registered.erase(framework->id());
   allocator->removeFramework(framework->id());
 
+  // Remove from 'frameworks.recovered' if necessary.
+  if (frameworks.recovered.contains(framework->id())) {
+    frameworks.recovered.erase(framework->id());
+  }
+
   // The completedFramework buffer now owns the framework pointer.
   frameworks.completed.push_back(shared_ptr<Framework>(framework));
 }

http://git-wip-us.apache.org/repos/asf/mesos/blob/3c96ca49/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 0d054d0..996ff42 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -480,6 +480,7 @@ public:
       const std::vector<Resource>& checkpointedResources,
       const std::vector<ExecutorInfo>& executorInfos,
       const std::vector<Task>& tasks,
+      const std::vector<FrameworkInfo>& frameworks,
       const std::vector<Archive::Framework>& completedFrameworks,
       const std::string& version);
 
@@ -557,6 +558,7 @@ public:
       const std::vector<Resource>& checkpointedResources,
       const std::vector<ExecutorInfo>& executorInfos,
       const std::vector<Task>& tasks,
+      const std::vector<FrameworkInfo>& frameworks,
       const std::vector<Archive::Framework>& completedFrameworks,
       const std::string& version,
       const process::Future<bool>& readmit);
@@ -612,7 +614,8 @@ protected:
 
   void __reregisterSlave(
       Slave* slave,
-      const std::vector<Task>& tasks);
+      const std::vector<Task>& tasks,
+      const std::vector<FrameworkInfo>& frameworks);
 
   // 'authenticate' is the future returned by the authenticator.
   void _authenticate(
@@ -1674,6 +1677,11 @@ private:
       : completed(masterFlags.max_completed_frameworks) {}
 
     hashmap<FrameworkID, Framework*> registered;
+
+    // 'Recovered' contains 'FrameworkInfo's for frameworks which
+    // would otherwise be unknown during recovery after master
+    // failover.
+    hashmap<FrameworkID, FrameworkInfo> recovered;
     boost::circular_buffer<std::shared_ptr<Framework>> completed;
 
     // Principals of frameworks keyed by PID.

http://git-wip-us.apache.org/repos/asf/mesos/blob/3c96ca49/src/messages/messages.proto
----------------------------------------------------------------------
diff --git a/src/messages/messages.proto b/src/messages/messages.proto
index 6d7eccc..f2177ae 100644
--- a/src/messages/messages.proto
+++ b/src/messages/messages.proto
@@ -418,6 +418,7 @@ message ReregisterSlaveMessage {
 
   repeated ExecutorInfo executor_infos = 4;
   repeated Task tasks = 3;
+  repeated FrameworkInfo frameworks = 8;
   repeated Archive.Framework completed_frameworks = 5;
 
   // NOTE: This is a hack for the master to detect the agent's

http://git-wip-us.apache.org/repos/asf/mesos/blob/3c96ca49/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index de4cb5a..066c0ee 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -1408,10 +1408,11 @@ void Slave::doReliableRegistration(Duration maxBackoff)
     message.mutable_slave()->CopyFrom(info);
 
     foreachvalue (Framework* framework, frameworks) {
+      message.add_frameworks()->CopyFrom(framework->info);
+
       // TODO(bmahler): We need to send the executors for these
       // pending tasks, and we need to send exited events if they
       // cannot be launched: MESOS-1715 MESOS-1720.
-
       typedef hashmap<TaskID, TaskInfo> TaskMap;
       foreachvalue (const TaskMap& tasks, framework->pending) {
         foreachvalue (const TaskInfo& task, tasks) {