You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 05:14:45 UTC

svn commit: r1131518 - in /incubator/mesos/trunk: include/nexus_types.hpp src/Makefile.in src/master.cpp src/master.hpp src/master_main.cpp src/nexus_exec.cpp src/nexus_sched.cpp src/nexus_types.cpp src/slave.cpp src/slave.hpp src/slave_main.cpp

Author: benh
Date: Sun Jun  5 03:14:44 2011
New Revision: 1131518

URL: http://svn.apache.org/viewvc?rev=1131518&view=rev
Log:
Master now registers with ZooKeeper. Slaves find master in Zookeeper. If new master elected, slaves switch over to it

Added:
    incubator/mesos/trunk/src/nexus_types.cpp
Modified:
    incubator/mesos/trunk/include/nexus_types.hpp
    incubator/mesos/trunk/src/Makefile.in
    incubator/mesos/trunk/src/master.cpp
    incubator/mesos/trunk/src/master.hpp
    incubator/mesos/trunk/src/master_main.cpp
    incubator/mesos/trunk/src/nexus_exec.cpp
    incubator/mesos/trunk/src/nexus_sched.cpp
    incubator/mesos/trunk/src/slave.cpp
    incubator/mesos/trunk/src/slave.hpp
    incubator/mesos/trunk/src/slave_main.cpp

Modified: incubator/mesos/trunk/include/nexus_types.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/nexus_types.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/include/nexus_types.hpp (original)
+++ incubator/mesos/trunk/include/nexus_types.hpp Sun Jun  5 03:14:44 2011
@@ -2,15 +2,22 @@
 #define __NEXUS_TYPES_HPP__
 
 #include "nexus.h"
+#include <cstdlib>
+#include <sstream>
 
 namespace nexus {
 
 typedef framework_id FrameworkID; // Unique within master
 typedef task_id TaskID;           // Unique within framework
-typedef slave_id SlaveID;         // Unique within master
+typedef std::string SlaveID;         // Unique within master
 typedef offer_id OfferID;         // Unique within master
 typedef task_state TaskState;
 
+slave_id slaveID_CPP2C(SlaveID sid);
+SlaveID slaveID_C2CPP(slave_id sid);
+
 }
 
+
+
 #endif /* __NEXUS_TYPES_HPP__ */

Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun  5 03:14:44 2011
@@ -88,7 +88,7 @@ NEXUS_LIBS = $(SCHED_LIB) $(EXEC_LIB) $(
 MASTER_OBJ = master.o allocator_factory.o simple_allocator.o
 SLAVE_OBJ = slave.o launcher.o isolation_module_factory.o \
 	    process_based_isolation_module.o
-COMMON_OBJ = fatal.o hash_pid.o messages.o getleader.o
+COMMON_OBJ = fatal.o hash_pid.o messages.o getleader.o nexus_types.o
 EXEC_LIB_OBJ = nexus_exec.o
 SCHED_LIB_OBJ = nexus_sched.o nexus_local.o params.o
 TEST_OBJ = tests/main.o tests/test_master.o tests/test_resources.o

Modified: incubator/mesos/trunk/src/master.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.cpp (original)
+++ incubator/mesos/trunk/src/master.cpp Sun Jun  5 03:14:44 2011
@@ -110,20 +110,22 @@ public:
 }
 
 
-Master::Master()
-  : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
-    allocatorType("simple")
+Master::Master(const bool ft, const string zk)
+  : isFT(ft), zkserver(zk), leaderDetector(NULL), nextFrameworkId(0), nextSlaveId("0"), 
+    nextSlotOfferId(0), allocatorType("simple")
 {}
 
 
-Master::Master(const string& _allocatorType)
-  : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
-    allocatorType(_allocatorType)
+Master::Master(const string& _allocatorType, const bool ft, const string zk)
+  : isFT(ft), zkserver(zk), leaderDetector(NULL), nextFrameworkId(0), nextSlaveId("0"), 
+    nextSlotOfferId(0), allocatorType(_allocatorType)
 {}
                    
 
 Master::~Master()
 {
+  if (isFT && leaderDetector!=NULL)
+    delete leaderDetector;
   LOG(INFO) << "Shutting down master";
   delete allocator;
   foreachpair (_, Framework *framework, frameworks) {
@@ -237,6 +239,13 @@ void Master::operator () ()
 {
   LOG(INFO) << "Master started at " << self();
 
+  if (isFT) {
+    LOG(INFO) << "Conencting to ZooKeeper at "<<zkserver;
+    ostringstream lpid;
+    lpid<<self();
+    leaderDetector = new LeaderDetector(zkserver, true, lpid.str());
+  }
+
   allocator = createAllocator();
   if (!allocator)
     LOG(FATAL) << "Unrecognized allocator type: " << allocatorType;
@@ -338,7 +347,11 @@ void Master::operator () ()
     }
 
     case S2M_REGISTER_SLAVE: {
-      Slave *slave = new Slave(from(), nextSlaveId++);
+      stringstream ss;
+      int nsi = atoi(nextSlaveId.c_str());
+      ss<<nsi++;
+      nextSlaveId = ss.str();
+      Slave *slave = new Slave(from(), nextSlaveId);
       unpack<S2M_REGISTER_SLAVE>(slave->hostname, slave->publicDns,
           slave->resources);
       LOG(INFO) << "Registering " << slave << " at " << slave->pid;

Modified: incubator/mesos/trunk/src/master.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.hpp (original)
+++ incubator/mesos/trunk/src/master.hpp Sun Jun  5 03:14:44 2011
@@ -27,6 +27,7 @@
 #include "messages.hpp"
 #include "params.hpp"
 #include "resources.hpp"
+#include "getleader.hpp"
 
 namespace nexus { namespace internal { namespace master {
 
@@ -272,6 +273,9 @@ enum TaskRemovalReason
 class Master : public Tuple<Process>
 {
 protected:
+  const bool isFT;
+  string zkserver;
+  LeaderDetector *leaderDetector;
   unordered_map<FrameworkID, Framework *> frameworks;
   unordered_map<SlaveID, Slave *> slaves;
   unordered_map<OfferID, SlotOffer *> slotOffers;
@@ -287,9 +291,9 @@ protected:
   Allocator *allocator;
 
 public:
-  Master();
+  Master(const bool =false, const string ="");
 
-  Master(const string& _allocatorType);
+  Master(const string& _allocatorType, const bool =false, const string ="");
   
   ~Master();
 

Modified: incubator/mesos/trunk/src/master_main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master_main.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master_main.cpp (original)
+++ incubator/mesos/trunk/src/master_main.cpp Sun Jun  5 03:14:44 2011
@@ -11,7 +11,7 @@ using namespace nexus::internal::master;
 void usage(const char* programName)
 {
   cerr << "Usage: " << programName
-       << " [--port PORT] [--allocator ALLOCATOR] [--quiet]"
+       << " [--port PORT] [--allocator ALLOCATOR] [--fault-tolerant ZOOKEEPERSERVER] [--quiet]"
        << endl;
 }
 
@@ -26,15 +26,18 @@ int main (int argc, char **argv)
   option options[] = {
     {"allocator", required_argument, 0, 'a'},
     {"port", required_argument, 0, 'p'},
+    {"fault-tolerant", required_argument, 0, 'f'},
     {"quiet", no_argument, 0, 'q'},
   };
 
+  bool isFT = false;
+  string zkserver = "";
   bool quiet = false;
   string allocator = "simple";
 
   int opt;
   int index;
-  while ((opt = getopt_long(argc, argv, "a:p:q", options, &index)) != -1) {
+  while ((opt = getopt_long(argc, argv, "a:p:f:q", options, &index)) != -1) {
     switch (opt) {
       case 'a':
         allocator = optarg;
@@ -42,6 +45,10 @@ int main (int argc, char **argv)
       case 'p':
         setenv("LIBPROCESS_PORT", optarg, 1);
         break;
+      case 'f':
+        isFT = true;
+	zkserver = optarg;
+        break;
       case 'q':
         quiet = true;
         break;
@@ -62,8 +69,11 @@ int main (int argc, char **argv)
 
   LOG(INFO) << "Build: " << BUILD_DATE << " by " << BUILD_USER;
   LOG(INFO) << "Starting Nexus master";
-  PID master = Process::spawn(new Master(allocator));
+  if (isFT)
+    LOG(INFO) << "Nexus in fault-tolerant mode";
+  PID master = Process::spawn(new Master(allocator, isFT, zkserver));
 
+#undef NEXUS_WEBUI
 #ifdef NEXUS_WEBUI
   startMasterWebUI(master);
 #endif

Modified: incubator/mesos/trunk/src/nexus_exec.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_exec.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/nexus_exec.cpp (original)
+++ incubator/mesos/trunk/src/nexus_exec.cpp Sun Jun  5 03:14:44 2011
@@ -264,7 +264,7 @@ public:
   virtual void init(const ExecutorArgs& args)
   {
     exec->init(exec,
-               args.slaveId,
+               slaveID_CPP2C(args.slaveId),
                args.frameworkId,
                args.frameworkName.c_str(),
                args.data.data(),
@@ -277,7 +277,7 @@ public:
     Params paramsObj(task.params);
     string paramsStr = paramsObj.str();
     nexus_task_desc td = { task.taskId,
-                           task.slaveId,
+                           slaveID_CPP2C(task.slaveId),
                            task.name.c_str(),
                            paramsStr.c_str(),
                            task.arg.data(),
@@ -292,7 +292,7 @@ public:
   
   virtual void frameworkMessage(const FrameworkMessage& message)
   {
-    nexus_framework_message msg = { message.slaveId,
+    nexus_framework_message msg = { slaveID_CPP2C(message.slaveId),
                                     message.taskId,
                                     message.data.data(),
                                     message.data.size() };
@@ -348,7 +348,7 @@ int nexus_exec_send_message(struct nexus
   }
 
   string data((char*) msg->data, msg->data_len);
-  FrameworkMessage message(msg->sid, msg->tid, data);
+  FrameworkMessage message(slaveID_C2CPP(msg->sid), msg->tid, data);
 
   _executor->sendFrameworkMessage(message);
 

Modified: incubator/mesos/trunk/src/nexus_sched.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_sched.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/nexus_sched.cpp (original)
+++ incubator/mesos/trunk/src/nexus_sched.cpp Sun Jun  5 03:14:44 2011
@@ -504,7 +504,7 @@ public:
     // Create C offer structs
     nexus_slot* c_offers = new nexus_slot[offers.size()];
     for (size_t i = 0; i < offers.size(); i++) {
-      nexus_slot offer = { offers[i].slaveId,
+      nexus_slot offer = { slaveID_CPP2C(offers[i].slaveId),
                            offers[i].host.c_str(),
                            paramStrs[i].c_str() };
       c_offers[i] = offer;
@@ -530,7 +530,7 @@ public:
 
   virtual void frameworkMessage(SchedulerDriver*, FrameworkMessage &message)
   {
-    nexus_framework_message c_message = { message.slaveId,
+    nexus_framework_message c_message = { slaveID_CPP2C(message.slaveId),
                                           message.taskId,
                                           message.data.data(),
                                           message.data.size() };
@@ -539,7 +539,7 @@ public:
 
   virtual void slaveLost(SchedulerDriver*, SlaveID sid)
   {
-    sched->slave_lost(sched, sid);
+    sched->slave_lost(sched, slaveID_CPP2C(sid));
   }
 
   virtual void error(SchedulerDriver*, int code, const std::string &message)
@@ -686,7 +686,7 @@ int nexus_sched_send_message(struct nexu
     return -1;
   }
 
-  FrameworkMessage message(msg->sid, msg->tid,
+  FrameworkMessage message(slaveID_C2CPP(msg->sid), msg->tid,
                            string((char*) msg->data, msg->data_len));
 
   CScheduler* csi = lookup(sched);
@@ -750,7 +750,7 @@ int nexus_sched_reply_to_offer(struct ne
     string taskArg((char*) tasks[i].arg, tasks[i].arg_len);
 
     wrapped_tasks[i] = TaskDescription(tasks[i].tid,
-                                       tasks[i].sid,
+                                       slaveID_C2CPP(tasks[i].sid),
                                        string(tasks[i].name),
                                        params,
                                        taskArg);

Added: incubator/mesos/trunk/src/nexus_types.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_types.cpp?rev=1131518&view=auto
==============================================================================
--- incubator/mesos/trunk/src/nexus_types.cpp (added)
+++ incubator/mesos/trunk/src/nexus_types.cpp Sun Jun  5 03:14:44 2011
@@ -0,0 +1,17 @@
+#include <nexus_types.hpp>
+#include <sstream>
+#include <string>
+
+namespace nexus { 
+
+slave_id slaveID_CPP2C(SlaveID sid) {
+  return atoi(sid.c_str());
+}
+    
+SlaveID slaveID_C2CPP(slave_id sid) {
+  std::stringstream ss;
+  ss<<sid;
+  return ss.str();
+}
+    
+}

Modified: incubator/mesos/trunk/src/slave.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave.cpp (original)
+++ incubator/mesos/trunk/src/slave.cpp Sun Jun  5 03:14:44 2011
@@ -58,16 +58,19 @@ public:
 } /* namespace */
 
 
-Slave::Slave(const PID &_master, Resources _resources, bool _local)
-  : master(_master), resources(_resources), local(_local), id(-1),
-    isolationType("process"), isolationModule(NULL)
+Slave::Slave(const PID &_master, Resources _resources, bool _local, 
+	     bool ft, string zk)
+  : isFT(ft), zkserver(zk), leaderDetector(NULL), master(_master), 
+    resources(_resources), local(_local), id("-1"),
+    isolationType("process"), isolationModule(NULL), slaveLeaderListener(this, getPID())
 {}
 
 
 Slave::Slave(const PID &_master, Resources _resources, bool _local,
-    const string& _isolationType)
-  : master(_master), resources(_resources), local(_local), id(-1),
-    isolationType(_isolationType), isolationModule(NULL)
+	     const string& _isolationType, bool ft, string zk)
+  : isFT(ft), zkserver(zk), leaderDetector(NULL), master(_master), 
+    resources(_resources), local(_local), id("-1"),
+    isolationType(_isolationType), isolationModule(NULL), slaveLeaderListener(this, getPID())
 {}
 
 
@@ -104,6 +107,20 @@ void Slave::operator () ()
 {
   LOG(INFO) << "Slave started at " << self();
 
+  if (isFT) {
+    LOG(INFO) << "Connecting to ZooKeeper at " << zkserver;
+    leaderDetector = new LeaderDetector(zkserver, false, "", NULL);
+    leaderDetector->setListener(&slaveLeaderListener); // use this instead of constructor to avoid race condition
+
+    pair<string,string> zkleader = leaderDetector->getCurrentLeader();
+    LOG(INFO) << "Detected leader at " << zkleader.second <<" with ephemeral id:"<<zkleader.first;
+
+    istringstream iss(zkleader.second);
+    if (!(iss >> master)) {
+      cerr << "Failed to resolve master PID " << zkleader.second << endl;
+    }    
+  }
+
   // Get our hostname
   char buf[256];
   gethostname(buf, sizeof(buf));
@@ -269,10 +286,10 @@ void Slave::operator () ()
 	  // TODO: Fault tolerance!
           LOG(ERROR) << "Master disconnected! Committing suicide ...";
 	  // TODO(matei): Add support for factory style destroy of objects!
-	  if (isolationModule != NULL)
-	    delete isolationModule;
-	  // TODO: Shut down executors?
-	  return;
+	  // if (isolationModule != NULL)
+	  //   delete isolationModule;
+	  // // TODO: Shut down executors?
+	  // return;
 	}
 
         foreachpair (_, Executor *ex, executors) {
@@ -297,6 +314,7 @@ void Slave::operator () ()
         return;
       }
 
+
       case S2S_SHUTDOWN: {
         LOG(INFO) << "Asked to shut down by " << from();
 	// TODO(matei): Add support for factory style destroy of objects!
@@ -306,6 +324,23 @@ void Slave::operator () ()
         return;
       }
 
+      case LE2S_NEWLEADER: {
+        LOG(INFO) << "Slave got notified of new leader " << from();
+	string newLeader;
+        unpack<LE2S_NEWLEADER>(newLeader);
+	istringstream iss(newLeader);
+	if (!(iss >> master)) {
+	  cerr << "Failed to resolve master PID " << newLeader << endl;
+	  break;
+	}    
+	
+	LOG(INFO) << "Connecting to Nexus master at " << master;
+	link(master);
+	send(master, pack<S2M_REGISTER_SLAVE>(hostname, publicDns, resources));
+	
+	break;
+      }
+
       default: {
         LOG(ERROR) << "Received unknown message ID " << msgid()
                    << " from " << from();

Modified: incubator/mesos/trunk/src/slave.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave.hpp (original)
+++ incubator/mesos/trunk/src/slave.hpp Sun Jun  5 03:14:44 2011
@@ -34,6 +34,7 @@
 #include "params.hpp"
 #include "resources.hpp"
 #include "slave_state.hpp"
+#include "getleader.hpp"
 
 namespace nexus { namespace internal { namespace slave {
 
@@ -176,6 +177,9 @@ public:
   typedef unordered_map<FrameworkID, Framework*> FrameworkMap;
   typedef unordered_map<FrameworkID, Executor*> ExecutorMap;
   
+  const bool isFT;
+  string zkserver;
+  LeaderDetector *leaderDetector;
   PID master;
   SlaveID id;
   Resources resources;
@@ -184,12 +188,39 @@ public:
   ExecutorMap executors;  // Invariant: framework will exist if executor exists
   string isolationType;
   IsolationModule *isolationModule;
+
+  
+  class SlaveLeaderListener;
+  friend class SlaveLeaderListener;
+
+  class SlaveLeaderListener : public LeaderListener {
+  public:
+    // TODO(alig): make thread safe
+    SlaveLeaderListener(Slave *s, PID pp) : parent(s), parentPID(pp) {}
+    
+    virtual void newLeaderElected(string zkId, string pidStr) {
+      if (zkId!="") {
+	LOG(INFO) << "Leader listener detected leader at " << pidStr <<" with ephemeral id:"<<zkId;
+	
+	parent->zkserver = pidStr;
+
+	LOG(INFO) << "Sending message to parent "<<parentPID<<" about new leader";
+	parent->send(parentPID, parent->pack<LE2S_NEWLEADER>(pidStr));
+
+      }
+    }
+
+  private:
+    Slave *parent;
+    PID parentPID;
+  } slaveLeaderListener;
+
   
 public:
-  Slave(const PID &_master, Resources resources, bool _local);
+  Slave(const PID &_master, Resources resources, bool _local, bool =false, string ="");
 
   Slave(const PID &_master, Resources resources, bool _local,
-        const string& isolationType);
+        const string& isolationType, bool =false, string ="");
 
   virtual ~Slave();
 

Modified: incubator/mesos/trunk/src/slave_main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave_main.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave_main.cpp (original)
+++ incubator/mesos/trunk/src/slave_main.cpp Sun Jun  5 03:14:44 2011
@@ -10,7 +10,7 @@ using namespace nexus::internal::slave;
 void usage(const char *programName)
 {
   cerr << "Usage: " << programName
-       << " [--cpus NUM] [--mem NUM] [--isolation TYPE] [--quiet] <master_pid>"
+       << " [--cpus NUM] [--mem NUM] [--isolation TYPE] [--server MASTER_PID][--fault-tolerant ZOOKEEPER_SERVER] [--quiet]"
        << endl;
 }
 
@@ -26,16 +26,22 @@ int main(int argc, char **argv)
     {"cpus", required_argument, 0, 'c'},
     {"mem", required_argument, 0, 'm'},
     {"isolation", required_argument, 0, 'i'},
+    {"server", optional_argument, 0, 's'},
+    {"fault-tolerant", optional_argument, 0, 'f'},
     {"quiet", no_argument, 0, 'q'},
   };
 
   Resources resources(1, 1 * Gigabyte);
+  bool isFT = false;
+  string zkserver = "";
   bool quiet = false;
   string isolation = "process";
 
   int opt;
   int index;
-  while ((opt = getopt_long(argc, argv, "c:m:i:q", options, &index)) != -1) {
+  string server="";
+  bool serveropt = false;
+  while ((opt = getopt_long(argc, argv, "c:m:i:f:s:q", options, &index)) != -1) {
     switch (opt) {
       case 'c':
 	resources.cpus = atoi(optarg);
@@ -46,6 +52,14 @@ int main(int argc, char **argv)
       case 'i':
 	isolation = optarg;
         break;
+      case 'f':
+        isFT = true;
+	zkserver = optarg;
+        break;
+      case 's':
+	serveropt = true;
+	server = optarg;
+        break;
       case 'q':
         quiet = true;
         break;
@@ -65,7 +79,12 @@ int main(int argc, char **argv)
   google::InitGoogleLogging(argv[0]);
 
   // Check that we have exactly one non-option argument (the master PID)
-  if (optind != argc - 1) {
+  // if (optind != argc - 1) {
+  //   usage(argv[0]);
+  //   exit(1);
+  // }
+  if (!isFT && !serveropt) {
+    cerr<<"Must either supply MASTER_PID or ZOOKEEPER_SERVER as parameters"<<endl;
     usage(argv[0]);
     exit(1);
   }
@@ -73,15 +92,18 @@ int main(int argc, char **argv)
   // Read and resolve the master PID
   PID master;
 
-  istringstream iss(argv[optind]);
-  if (!(iss >> master)) {
-    cerr << "Failed to resolve master PID " << argv[optind] << endl;
-    exit(1);
+  if (serveropt) {
+    LOG(INFO) << "Parsing MASTER_PID";
+    istringstream iss(server);
+    if (!(iss >> master)) {
+      cerr << "Failed to resolve master PID " << server << endl;
+      exit(1);
+    }
   }
 
   LOG(INFO) << "Build: " << BUILD_DATE << " by " << BUILD_USER;
   LOG(INFO) << "Starting Nexus slave";
-  PID slave = Process::spawn(new Slave(master, resources, false, isolation));
+  PID slave = Process::spawn(new Slave(master, resources, false, isolation, isFT, zkserver));
 
 #ifdef NEXUS_WEBUI
   startSlaveWebUI(slave);