You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 05:14:45 UTC
svn commit: r1131518 - in /incubator/mesos/trunk: include/nexus_types.hpp
src/Makefile.in src/master.cpp src/master.hpp src/master_main.cpp
src/nexus_exec.cpp src/nexus_sched.cpp src/nexus_types.cpp src/slave.cpp
src/slave.hpp src/slave_main.cpp
Author: benh
Date: Sun Jun 5 03:14:44 2011
New Revision: 1131518
URL: http://svn.apache.org/viewvc?rev=1131518&view=rev
Log:
Master now registers with ZooKeeper. Slaves find master in Zookeeper. If new master elected, slaves switch over to it
Added:
incubator/mesos/trunk/src/nexus_types.cpp
Modified:
incubator/mesos/trunk/include/nexus_types.hpp
incubator/mesos/trunk/src/Makefile.in
incubator/mesos/trunk/src/master.cpp
incubator/mesos/trunk/src/master.hpp
incubator/mesos/trunk/src/master_main.cpp
incubator/mesos/trunk/src/nexus_exec.cpp
incubator/mesos/trunk/src/nexus_sched.cpp
incubator/mesos/trunk/src/slave.cpp
incubator/mesos/trunk/src/slave.hpp
incubator/mesos/trunk/src/slave_main.cpp
Modified: incubator/mesos/trunk/include/nexus_types.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/include/nexus_types.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/include/nexus_types.hpp (original)
+++ incubator/mesos/trunk/include/nexus_types.hpp Sun Jun 5 03:14:44 2011
@@ -2,15 +2,22 @@
#define __NEXUS_TYPES_HPP__
#include "nexus.h"
+#include <cstdlib>
+#include <sstream>
namespace nexus {
typedef framework_id FrameworkID; // Unique within master
typedef task_id TaskID; // Unique within framework
-typedef slave_id SlaveID; // Unique within master
+typedef std::string SlaveID; // Unique within master
typedef offer_id OfferID; // Unique within master
typedef task_state TaskState;
+slave_id slaveID_CPP2C(SlaveID sid);
+SlaveID slaveID_C2CPP(slave_id sid);
+
}
+
+
#endif /* __NEXUS_TYPES_HPP__ */
Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun 5 03:14:44 2011
@@ -88,7 +88,7 @@ NEXUS_LIBS = $(SCHED_LIB) $(EXEC_LIB) $(
MASTER_OBJ = master.o allocator_factory.o simple_allocator.o
SLAVE_OBJ = slave.o launcher.o isolation_module_factory.o \
process_based_isolation_module.o
-COMMON_OBJ = fatal.o hash_pid.o messages.o getleader.o
+COMMON_OBJ = fatal.o hash_pid.o messages.o getleader.o nexus_types.o
EXEC_LIB_OBJ = nexus_exec.o
SCHED_LIB_OBJ = nexus_sched.o nexus_local.o params.o
TEST_OBJ = tests/main.o tests/test_master.o tests/test_resources.o
Modified: incubator/mesos/trunk/src/master.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.cpp (original)
+++ incubator/mesos/trunk/src/master.cpp Sun Jun 5 03:14:44 2011
@@ -110,20 +110,22 @@ public:
}
-Master::Master()
- : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
- allocatorType("simple")
+Master::Master(const bool ft, const string zk)
+ : isFT(ft), zkserver(zk), leaderDetector(NULL), nextFrameworkId(0), nextSlaveId("0"),
+ nextSlotOfferId(0), allocatorType("simple")
{}
-Master::Master(const string& _allocatorType)
- : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
- allocatorType(_allocatorType)
+Master::Master(const string& _allocatorType, const bool ft, const string zk)
+ : isFT(ft), zkserver(zk), leaderDetector(NULL), nextFrameworkId(0), nextSlaveId("0"),
+ nextSlotOfferId(0), allocatorType(_allocatorType)
{}
Master::~Master()
{
+ if (isFT && leaderDetector!=NULL)
+ delete leaderDetector;
LOG(INFO) << "Shutting down master";
delete allocator;
foreachpair (_, Framework *framework, frameworks) {
@@ -237,6 +239,13 @@ void Master::operator () ()
{
LOG(INFO) << "Master started at " << self();
+ if (isFT) {
+ LOG(INFO) << "Conencting to ZooKeeper at "<<zkserver;
+ ostringstream lpid;
+ lpid<<self();
+ leaderDetector = new LeaderDetector(zkserver, true, lpid.str());
+ }
+
allocator = createAllocator();
if (!allocator)
LOG(FATAL) << "Unrecognized allocator type: " << allocatorType;
@@ -338,7 +347,11 @@ void Master::operator () ()
}
case S2M_REGISTER_SLAVE: {
- Slave *slave = new Slave(from(), nextSlaveId++);
+ stringstream ss;
+ int nsi = atoi(nextSlaveId.c_str());
+ ss<<nsi++;
+ nextSlaveId = ss.str();
+ Slave *slave = new Slave(from(), nextSlaveId);
unpack<S2M_REGISTER_SLAVE>(slave->hostname, slave->publicDns,
slave->resources);
LOG(INFO) << "Registering " << slave << " at " << slave->pid;
Modified: incubator/mesos/trunk/src/master.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.hpp (original)
+++ incubator/mesos/trunk/src/master.hpp Sun Jun 5 03:14:44 2011
@@ -27,6 +27,7 @@
#include "messages.hpp"
#include "params.hpp"
#include "resources.hpp"
+#include "getleader.hpp"
namespace nexus { namespace internal { namespace master {
@@ -272,6 +273,9 @@ enum TaskRemovalReason
class Master : public Tuple<Process>
{
protected:
+ const bool isFT;
+ string zkserver;
+ LeaderDetector *leaderDetector;
unordered_map<FrameworkID, Framework *> frameworks;
unordered_map<SlaveID, Slave *> slaves;
unordered_map<OfferID, SlotOffer *> slotOffers;
@@ -287,9 +291,9 @@ protected:
Allocator *allocator;
public:
- Master();
+ Master(const bool =false, const string ="");
- Master(const string& _allocatorType);
+ Master(const string& _allocatorType, const bool =false, const string ="");
~Master();
Modified: incubator/mesos/trunk/src/master_main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master_main.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master_main.cpp (original)
+++ incubator/mesos/trunk/src/master_main.cpp Sun Jun 5 03:14:44 2011
@@ -11,7 +11,7 @@ using namespace nexus::internal::master;
void usage(const char* programName)
{
cerr << "Usage: " << programName
- << " [--port PORT] [--allocator ALLOCATOR] [--quiet]"
+ << " [--port PORT] [--allocator ALLOCATOR] [--fault-tolerant ZOOKEEPERSERVER] [--quiet]"
<< endl;
}
@@ -26,15 +26,18 @@ int main (int argc, char **argv)
option options[] = {
{"allocator", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'},
+ {"fault-tolerant", required_argument, 0, 'f'},
{"quiet", no_argument, 0, 'q'},
};
+ bool isFT = false;
+ string zkserver = "";
bool quiet = false;
string allocator = "simple";
int opt;
int index;
- while ((opt = getopt_long(argc, argv, "a:p:q", options, &index)) != -1) {
+ while ((opt = getopt_long(argc, argv, "a:p:f:q", options, &index)) != -1) {
switch (opt) {
case 'a':
allocator = optarg;
@@ -42,6 +45,10 @@ int main (int argc, char **argv)
case 'p':
setenv("LIBPROCESS_PORT", optarg, 1);
break;
+ case 'f':
+ isFT = true;
+ zkserver = optarg;
+ break;
case 'q':
quiet = true;
break;
@@ -62,8 +69,11 @@ int main (int argc, char **argv)
LOG(INFO) << "Build: " << BUILD_DATE << " by " << BUILD_USER;
LOG(INFO) << "Starting Nexus master";
- PID master = Process::spawn(new Master(allocator));
+ if (isFT)
+ LOG(INFO) << "Nexus in fault-tolerant mode";
+ PID master = Process::spawn(new Master(allocator, isFT, zkserver));
+#undef NEXUS_WEBUI
#ifdef NEXUS_WEBUI
startMasterWebUI(master);
#endif
Modified: incubator/mesos/trunk/src/nexus_exec.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_exec.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/nexus_exec.cpp (original)
+++ incubator/mesos/trunk/src/nexus_exec.cpp Sun Jun 5 03:14:44 2011
@@ -264,7 +264,7 @@ public:
virtual void init(const ExecutorArgs& args)
{
exec->init(exec,
- args.slaveId,
+ slaveID_CPP2C(args.slaveId),
args.frameworkId,
args.frameworkName.c_str(),
args.data.data(),
@@ -277,7 +277,7 @@ public:
Params paramsObj(task.params);
string paramsStr = paramsObj.str();
nexus_task_desc td = { task.taskId,
- task.slaveId,
+ slaveID_CPP2C(task.slaveId),
task.name.c_str(),
paramsStr.c_str(),
task.arg.data(),
@@ -292,7 +292,7 @@ public:
virtual void frameworkMessage(const FrameworkMessage& message)
{
- nexus_framework_message msg = { message.slaveId,
+ nexus_framework_message msg = { slaveID_CPP2C(message.slaveId),
message.taskId,
message.data.data(),
message.data.size() };
@@ -348,7 +348,7 @@ int nexus_exec_send_message(struct nexus
}
string data((char*) msg->data, msg->data_len);
- FrameworkMessage message(msg->sid, msg->tid, data);
+ FrameworkMessage message(slaveID_C2CPP(msg->sid), msg->tid, data);
_executor->sendFrameworkMessage(message);
Modified: incubator/mesos/trunk/src/nexus_sched.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_sched.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/nexus_sched.cpp (original)
+++ incubator/mesos/trunk/src/nexus_sched.cpp Sun Jun 5 03:14:44 2011
@@ -504,7 +504,7 @@ public:
// Create C offer structs
nexus_slot* c_offers = new nexus_slot[offers.size()];
for (size_t i = 0; i < offers.size(); i++) {
- nexus_slot offer = { offers[i].slaveId,
+ nexus_slot offer = { slaveID_CPP2C(offers[i].slaveId),
offers[i].host.c_str(),
paramStrs[i].c_str() };
c_offers[i] = offer;
@@ -530,7 +530,7 @@ public:
virtual void frameworkMessage(SchedulerDriver*, FrameworkMessage &message)
{
- nexus_framework_message c_message = { message.slaveId,
+ nexus_framework_message c_message = { slaveID_CPP2C(message.slaveId),
message.taskId,
message.data.data(),
message.data.size() };
@@ -539,7 +539,7 @@ public:
virtual void slaveLost(SchedulerDriver*, SlaveID sid)
{
- sched->slave_lost(sched, sid);
+ sched->slave_lost(sched, slaveID_CPP2C(sid));
}
virtual void error(SchedulerDriver*, int code, const std::string &message)
@@ -686,7 +686,7 @@ int nexus_sched_send_message(struct nexu
return -1;
}
- FrameworkMessage message(msg->sid, msg->tid,
+ FrameworkMessage message(slaveID_C2CPP(msg->sid), msg->tid,
string((char*) msg->data, msg->data_len));
CScheduler* csi = lookup(sched);
@@ -750,7 +750,7 @@ int nexus_sched_reply_to_offer(struct ne
string taskArg((char*) tasks[i].arg, tasks[i].arg_len);
wrapped_tasks[i] = TaskDescription(tasks[i].tid,
- tasks[i].sid,
+ slaveID_C2CPP(tasks[i].sid),
string(tasks[i].name),
params,
taskArg);
Added: incubator/mesos/trunk/src/nexus_types.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_types.cpp?rev=1131518&view=auto
==============================================================================
--- incubator/mesos/trunk/src/nexus_types.cpp (added)
+++ incubator/mesos/trunk/src/nexus_types.cpp Sun Jun 5 03:14:44 2011
@@ -0,0 +1,17 @@
+#include <nexus_types.hpp>
+#include <sstream>
+#include <string>
+
+namespace nexus {
+
+slave_id slaveID_CPP2C(SlaveID sid) {
+ return atoi(sid.c_str());
+}
+
+SlaveID slaveID_C2CPP(slave_id sid) {
+ std::stringstream ss;
+ ss<<sid;
+ return ss.str();
+}
+
+}
Modified: incubator/mesos/trunk/src/slave.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave.cpp (original)
+++ incubator/mesos/trunk/src/slave.cpp Sun Jun 5 03:14:44 2011
@@ -58,16 +58,19 @@ public:
} /* namespace */
-Slave::Slave(const PID &_master, Resources _resources, bool _local)
- : master(_master), resources(_resources), local(_local), id(-1),
- isolationType("process"), isolationModule(NULL)
+Slave::Slave(const PID &_master, Resources _resources, bool _local,
+ bool ft, string zk)
+ : isFT(ft), zkserver(zk), leaderDetector(NULL), master(_master),
+ resources(_resources), local(_local), id("-1"),
+ isolationType("process"), isolationModule(NULL), slaveLeaderListener(this, getPID())
{}
Slave::Slave(const PID &_master, Resources _resources, bool _local,
- const string& _isolationType)
- : master(_master), resources(_resources), local(_local), id(-1),
- isolationType(_isolationType), isolationModule(NULL)
+ const string& _isolationType, bool ft, string zk)
+ : isFT(ft), zkserver(zk), leaderDetector(NULL), master(_master),
+ resources(_resources), local(_local), id("-1"),
+ isolationType(_isolationType), isolationModule(NULL), slaveLeaderListener(this, getPID())
{}
@@ -104,6 +107,20 @@ void Slave::operator () ()
{
LOG(INFO) << "Slave started at " << self();
+ if (isFT) {
+ LOG(INFO) << "Connecting to ZooKeeper at " << zkserver;
+ leaderDetector = new LeaderDetector(zkserver, false, "", NULL);
+ leaderDetector->setListener(&slaveLeaderListener); // use this instead of constructor to avoid race condition
+
+ pair<string,string> zkleader = leaderDetector->getCurrentLeader();
+ LOG(INFO) << "Detected leader at " << zkleader.second <<" with ephemeral id:"<<zkleader.first;
+
+ istringstream iss(zkleader.second);
+ if (!(iss >> master)) {
+ cerr << "Failed to resolve master PID " << zkleader.second << endl;
+ }
+ }
+
// Get our hostname
char buf[256];
gethostname(buf, sizeof(buf));
@@ -269,10 +286,10 @@ void Slave::operator () ()
// TODO: Fault tolerance!
LOG(ERROR) << "Master disconnected! Committing suicide ...";
// TODO(matei): Add support for factory style destroy of objects!
- if (isolationModule != NULL)
- delete isolationModule;
- // TODO: Shut down executors?
- return;
+ // if (isolationModule != NULL)
+ // delete isolationModule;
+ // // TODO: Shut down executors?
+ // return;
}
foreachpair (_, Executor *ex, executors) {
@@ -297,6 +314,7 @@ void Slave::operator () ()
return;
}
+
case S2S_SHUTDOWN: {
LOG(INFO) << "Asked to shut down by " << from();
// TODO(matei): Add support for factory style destroy of objects!
@@ -306,6 +324,23 @@ void Slave::operator () ()
return;
}
+ case LE2S_NEWLEADER: {
+ LOG(INFO) << "Slave got notified of new leader " << from();
+ string newLeader;
+ unpack<LE2S_NEWLEADER>(newLeader);
+ istringstream iss(newLeader);
+ if (!(iss >> master)) {
+ cerr << "Failed to resolve master PID " << newLeader << endl;
+ break;
+ }
+
+ LOG(INFO) << "Connecting to Nexus master at " << master;
+ link(master);
+ send(master, pack<S2M_REGISTER_SLAVE>(hostname, publicDns, resources));
+
+ break;
+ }
+
default: {
LOG(ERROR) << "Received unknown message ID " << msgid()
<< " from " << from();
Modified: incubator/mesos/trunk/src/slave.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave.hpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave.hpp (original)
+++ incubator/mesos/trunk/src/slave.hpp Sun Jun 5 03:14:44 2011
@@ -34,6 +34,7 @@
#include "params.hpp"
#include "resources.hpp"
#include "slave_state.hpp"
+#include "getleader.hpp"
namespace nexus { namespace internal { namespace slave {
@@ -176,6 +177,9 @@ public:
typedef unordered_map<FrameworkID, Framework*> FrameworkMap;
typedef unordered_map<FrameworkID, Executor*> ExecutorMap;
+ const bool isFT;
+ string zkserver;
+ LeaderDetector *leaderDetector;
PID master;
SlaveID id;
Resources resources;
@@ -184,12 +188,39 @@ public:
ExecutorMap executors; // Invariant: framework will exist if executor exists
string isolationType;
IsolationModule *isolationModule;
+
+
+ class SlaveLeaderListener;
+ friend class SlaveLeaderListener;
+
+ class SlaveLeaderListener : public LeaderListener {
+ public:
+ // TODO(alig): make thread safe
+ SlaveLeaderListener(Slave *s, PID pp) : parent(s), parentPID(pp) {}
+
+ virtual void newLeaderElected(string zkId, string pidStr) {
+ if (zkId!="") {
+ LOG(INFO) << "Leader listener detected leader at " << pidStr <<" with ephemeral id:"<<zkId;
+
+ parent->zkserver = pidStr;
+
+ LOG(INFO) << "Sending message to parent "<<parentPID<<" about new leader";
+ parent->send(parentPID, parent->pack<LE2S_NEWLEADER>(pidStr));
+
+ }
+ }
+
+ private:
+ Slave *parent;
+ PID parentPID;
+ } slaveLeaderListener;
+
public:
- Slave(const PID &_master, Resources resources, bool _local);
+ Slave(const PID &_master, Resources resources, bool _local, bool =false, string ="");
Slave(const PID &_master, Resources resources, bool _local,
- const string& isolationType);
+ const string& isolationType, bool =false, string ="");
virtual ~Slave();
Modified: incubator/mesos/trunk/src/slave_main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/slave_main.cpp?rev=1131518&r1=1131517&r2=1131518&view=diff
==============================================================================
--- incubator/mesos/trunk/src/slave_main.cpp (original)
+++ incubator/mesos/trunk/src/slave_main.cpp Sun Jun 5 03:14:44 2011
@@ -10,7 +10,7 @@ using namespace nexus::internal::slave;
void usage(const char *programName)
{
cerr << "Usage: " << programName
- << " [--cpus NUM] [--mem NUM] [--isolation TYPE] [--quiet] <master_pid>"
+ << " [--cpus NUM] [--mem NUM] [--isolation TYPE] [--server MASTER_PID][--fault-tolerant ZOOKEEPER_SERVER] [--quiet]"
<< endl;
}
@@ -26,16 +26,22 @@ int main(int argc, char **argv)
{"cpus", required_argument, 0, 'c'},
{"mem", required_argument, 0, 'm'},
{"isolation", required_argument, 0, 'i'},
+ {"server", optional_argument, 0, 's'},
+ {"fault-tolerant", optional_argument, 0, 'f'},
{"quiet", no_argument, 0, 'q'},
};
Resources resources(1, 1 * Gigabyte);
+ bool isFT = false;
+ string zkserver = "";
bool quiet = false;
string isolation = "process";
int opt;
int index;
- while ((opt = getopt_long(argc, argv, "c:m:i:q", options, &index)) != -1) {
+ string server="";
+ bool serveropt = false;
+ while ((opt = getopt_long(argc, argv, "c:m:i:f:s:q", options, &index)) != -1) {
switch (opt) {
case 'c':
resources.cpus = atoi(optarg);
@@ -46,6 +52,14 @@ int main(int argc, char **argv)
case 'i':
isolation = optarg;
break;
+ case 'f':
+ isFT = true;
+ zkserver = optarg;
+ break;
+ case 's':
+ serveropt = true;
+ server = optarg;
+ break;
case 'q':
quiet = true;
break;
@@ -65,7 +79,12 @@ int main(int argc, char **argv)
google::InitGoogleLogging(argv[0]);
// Check that we have exactly one non-option argument (the master PID)
- if (optind != argc - 1) {
+ // if (optind != argc - 1) {
+ // usage(argv[0]);
+ // exit(1);
+ // }
+ if (!isFT && !serveropt) {
+ cerr<<"Must either supply MASTER_PID or ZOOKEEPER_SERVER as parameters"<<endl;
usage(argv[0]);
exit(1);
}
@@ -73,15 +92,18 @@ int main(int argc, char **argv)
// Read and resolve the master PID
PID master;
- istringstream iss(argv[optind]);
- if (!(iss >> master)) {
- cerr << "Failed to resolve master PID " << argv[optind] << endl;
- exit(1);
+ if (serveropt) {
+ LOG(INFO) << "Parsing MASTER_PID";
+ istringstream iss(server);
+ if (!(iss >> master)) {
+ cerr << "Failed to resolve master PID " << server << endl;
+ exit(1);
+ }
}
LOG(INFO) << "Build: " << BUILD_DATE << " by " << BUILD_USER;
LOG(INFO) << "Starting Nexus slave";
- PID slave = Process::spawn(new Slave(master, resources, false, isolation));
+ PID slave = Process::spawn(new Slave(master, resources, false, isolation, isFT, zkserver));
#ifdef NEXUS_WEBUI
startSlaveWebUI(slave);