You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 07:43:38 UTC
svn commit: r1131876 - in /incubator/mesos/trunk/src: conf/ conf/mesos.conf
configuration.cpp configuration.hpp master.cpp master.hpp master_main.cpp
nexus_local.cpp params.hpp tests/test_configuration.cpp
Author: benh
Date: Sun Jun 5 05:43:37 2011
New Revision: 1131876
URL: http://svn.apache.org/viewvc?rev=1131876&view=rev
Log:
Initial work on getting master to use new conf framework
Added:
incubator/mesos/trunk/src/conf/
incubator/mesos/trunk/src/conf/mesos.conf
Modified:
incubator/mesos/trunk/src/configuration.cpp
incubator/mesos/trunk/src/configuration.hpp
incubator/mesos/trunk/src/master.cpp
incubator/mesos/trunk/src/master.hpp
incubator/mesos/trunk/src/master_main.cpp
incubator/mesos/trunk/src/nexus_local.cpp
incubator/mesos/trunk/src/params.hpp
incubator/mesos/trunk/src/tests/test_configuration.cpp
Added: incubator/mesos/trunk/src/conf/mesos.conf
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/conf/mesos.conf?rev=1131876&view=auto
==============================================================================
(empty)
Modified: incubator/mesos/trunk/src/configuration.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/configuration.cpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/configuration.cpp (original)
+++ incubator/mesos/trunk/src/configuration.cpp Sun Jun 5 05:43:37 2011
@@ -5,9 +5,8 @@
#include <algorithm>
#include <iostream>
-#include <boost/foreach.hpp>
-
#include "configuration.hpp"
+#include "foreach.hpp"
#include "params.hpp"
extern char** environ; // libc's environment variable list; for some reason,
@@ -23,7 +22,7 @@ const char* Configuration::ENV_VAR_PREFI
void Configuration::validate()
{
- foreachpair(const string& key, const Option& opt, options) {
+ foreachpair (const string& key, const Option& opt, options) {
if (params.contains(key) && opt.validator && !opt.validator->isValid(params[key])) {
throw BadOptionValueException(params[key].c_str());
}
@@ -184,20 +183,20 @@ string Configuration::getUsage() const
{
const int PAD = 10;
const int PADEXTRA = string("--=VAL").size(); // adjust for "--" and "=VAL"
- string usage = "Parameters:\n\n";
+ string usage = "";
// get max key length
int maxLen = 0;
- foreachpair(const string& key, _, options) {
+ foreachpair (const string& key, _, options) {
maxLen = key.size() > maxLen ? key.length() : maxLen;
}
maxLen += PADEXTRA;
- foreachpair(const string& key, const Option& val, options) {
+ foreachpair (const string& key, const Option& val, options) {
string helpStr = val.helpString;
- if (val.defaultValue != "") { // add default value
- helpStr += " (default VAL=" + val.defaultValue + ")";
+ if (val.hasDefault) { // add default value
+ helpStr += " (default: " + val.defaultValue + ")";
}
usage += "--" + key + "=VAL";
@@ -220,29 +219,10 @@ string Configuration::getUsage() const
}
-int Configuration::addOption(string optName, const string& helpString)
-{
- std::transform(optName.begin(), optName.end(), optName.begin(), ::tolower);
- if (options.find(optName) != options.end())
- return -1;
- options[optName] = Option(helpString);
- return 0;
-}
-
-
-string Configuration::getOptionDefault(string optName) const
-{
- std::transform(optName.begin(), optName.end(), optName.begin(), ::tolower);
- if (options.find(optName) == options.end())
- return "";
- return options.find(optName)->second.defaultValue;
-}
-
-
vector<string> Configuration::getOptions() const
{
vector<string> ret;
- foreachpair(const string& key, _, options) {
+ foreachpair (const string& key, _, options) {
ret.push_back(key);
}
return ret;
Modified: incubator/mesos/trunk/src/configuration.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/configuration.hpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/configuration.hpp (original)
+++ incubator/mesos/trunk/src/configuration.hpp Sun Jun 5 05:43:37 2011
@@ -5,7 +5,6 @@
#include <fstream>
#include <string>
#include <map>
-#include <boost/any.hpp>
#include <glog/logging.h>
#include "params.hpp"
#include "foreach.hpp"
@@ -88,27 +87,31 @@ struct BadOptionValueException : std::ex
**/
struct Option {
Option(string _helpString) :
- helpString(_helpString), defaultValue(""), validator(NULL) {}
+ helpString(_helpString), hasDefault(false), validator(NULL) {}
-
- Option(string _helpString, string _defaultValue,
- const ValidatorBase& _validator) :
- helpString(_helpString), defaultValue(_defaultValue)
+ Option(string _helpString,
+ const ValidatorBase& _validator,
+ string _defaultValue)
+ : helpString(_helpString), hasDefault(true), defaultValue(_defaultValue)
{
validator = _validator.clone();
- }
-
+ }
- Option() : validator(NULL) {}
+ Option(string _helpString, const ValidatorBase& _validator)
+ : helpString(_helpString), hasDefault(false)
+ {
+ validator = _validator.clone();
+ }
+ Option() : hasDefault(false), validator(NULL) {}
- Option(const Option& opt) :
- helpString(opt.helpString), defaultValue(opt.defaultValue)
+ Option(const Option& opt)
+ : helpString(opt.helpString), hasDefault(opt.hasDefault),
+ defaultValue(opt.defaultValue)
{
validator = opt.validator == NULL ? NULL : opt.validator->clone();
}
-
Option &operator=(const Option& opt)
{
helpString = opt.helpString;
@@ -117,14 +120,13 @@ struct Option {
return *this;
}
-
~Option()
{
if (validator != 0) delete validator;
}
-
string helpString;
+ bool hasDefault;
string defaultValue;
ValidatorBase *validator;
};
@@ -194,7 +196,7 @@ public:
return -1;
ostringstream os;
os << defaultValue;
- options[optName] = Option(helpString, os.str(), Validator<T>());
+ options[optName] = Option(helpString, Validator<T>(), os.str());
if (!params.contains(optName)) // insert default value
params[optName] = os.str();
@@ -202,23 +204,26 @@ public:
return 0;
}
-
+
/**
- * Adds a registered option together with a help string
- * It's recommended to use the other version of this method,
- * which takes a default value.
+ * Adds a registered option together with a help string.
* @param optName name of the option, e.g. "home"
* @param helpString description of the option, may contain line breaks
+ * @param defaultValue default value of the option.
+ * The default option is put in the internal params,
+ * unless the option already has a value in params.
+ * Its type must support operator<<(ostream,...)
* @return 0 on success, -1 if option already exists
**/
- int addOption(string optName, const string& helpString);
-
- /**
- * Returns the default string value associated with an option.
- * @param optName name of the option, e.g. "home"
- * @return default value associated with optName
- **/
- string getOptionDefault(string optName) const;
+ template <class T>
+ int addOption(string optName, const string& helpString)
+ {
+ std::transform(optName.begin(), optName.end(), optName.begin(), ::tolower);
+ if (options.find(optName) != options.end())
+ return -1;
+ options[optName] = Option(helpString, Validator<T>());
+ return 0;
+ }
/**
Modified: incubator/mesos/trunk/src/master.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.cpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.cpp (original)
+++ incubator/mesos/trunk/src/master.cpp Sun Jun 5 05:43:37 2011
@@ -112,9 +112,19 @@ public:
}
-Master::Master(const string& _allocatorType)
- : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
- allocatorType(_allocatorType), masterId(0) {}
+Master::Master()
+ : nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0), masterId(0)
+{
+ allocatorType = "simple";
+}
+
+
+Master::Master(const Params& conf_)
+ : conf(conf_), nextFrameworkId(0), nextSlaveId(0), nextSlotOfferId(0),
+ masterId(0)
+{
+ allocatorType = conf.get("allocator", "simple");
+}
Master::~Master()
@@ -139,6 +149,12 @@ Master::~Master()
}
+void Master::registerOptions(Configuration* conf)
+{
+ conf->addOption<string>("allocator", "Allocation module to use", "simple");
+}
+
+
state::MasterState * Master::getState()
{
std::ostringstream oss;
Modified: incubator/mesos/trunk/src/master.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master.hpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master.hpp (original)
+++ incubator/mesos/trunk/src/master.hpp Sun Jun 5 05:43:37 2011
@@ -21,6 +21,8 @@
#include <boost/unordered_map.hpp>
#include <boost/unordered_set.hpp>
+#include "configuration.hpp"
+#include "fatal.hpp"
#include "foreach.hpp"
#include "hash_pid.hpp"
#include "master_state.hpp"
@@ -69,7 +71,7 @@ const int64_t MAX_MEM = 1024LL * 1024LL
const double HEARTBEAT_INTERVAL = 2;
// Acceptable time since we saw the last heartbeat (four heartbeats).
-const double HEARTBEAT_TIMEOUT = 15;
+const double HEARTBEAT_TIMEOUT = HEARTBEAT_INTERVAL * 4;
// Some forward declarations
class Slave;
@@ -265,6 +267,8 @@ enum TaskRemovalReason
class Master : public Tuple<ReliableProcess>
{
protected:
+ Params conf;
+
unordered_map<FrameworkID, Framework *> frameworks;
unordered_map<SlaveID, Slave *> slaves;
unordered_map<OfferID, SlotOffer *> slotOffers;
@@ -283,10 +287,14 @@ protected:
// will be this master's ZooKeeper ephemeral id
public:
- Master(const string& _allocatorType = "simple");
+ Master();
+
+ Master(const Params& conf_);
~Master();
+ static void registerOptions(Configuration* conf);
+
state::MasterState *getState();
OfferID makeOffer(Framework *framework,
Modified: incubator/mesos/trunk/src/master_main.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master_main.cpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master_main.cpp (original)
+++ incubator/mesos/trunk/src/master_main.cpp Sun Jun 5 05:43:37 2011
@@ -1,112 +1,87 @@
#include <getopt.h>
#include <libgen.h>
+#include "configuration.hpp"
#include "master.hpp"
#include "master_webui.hpp"
using std::cerr;
using std::endl;
-using boost::lexical_cast;
-using boost::bad_lexical_cast;
using namespace nexus::internal::master;
-void usage(const char* programName)
+void usage(const char* programName, const Configuration& conf)
{
cerr << "Usage: " << programName
- << " [--url URL]"
<< " [--port PORT]"
- << " [--webui-port PORT]"
+ << " [--url URL]"
<< " [--allocator ALLOCATOR]"
<< " [--quiet]" << endl
<< endl
<< "URL (used for contending to be a master) may be one of:" << endl
<< " zoo://host1:port1,host2:port2,..." << endl
<< " zoofile://file where file contains a host:port pair per line"
- << endl;
+ << endl << endl
+ << "Option details:" << endl
+ << conf.getUsage() << endl;
}
-int main (int argc, char **argv)
+int main(int argc, char **argv)
{
+ Configuration conf;
+ conf.addOption<string>("url", "URL used for contending to a master.");
+ conf.addOption<int>("port", "Port to listen on");
+ conf.addOption<bool>("quiet", "Do not log to stderr", false);
+ conf.addOption<string>("log_dir", "Where to place logs", "/tmp");
+ Master::registerOptions(&conf);
+
if (argc == 2 && string("--help") == argv[1]) {
- usage(argv[0]);
+ usage(argv[0], conf);
exit(1);
}
- option options[] = {
- {"url", required_argument, 0, 'u'},
- {"allocator", required_argument, 0, 'a'},
- {"port", required_argument, 0, 'p'},
- {"webui-port", required_argument, 0, 'w'},
- {"quiet", no_argument, 0, 'q'},
- };
-
- string url = "";
- string allocator = "simple";
- char* webuiPortStr = "8080"; // C string because it is sent to python C API
- bool quiet = false;
-
- int opt;
- int index;
- while ((opt = getopt_long(argc, argv, "u:a:p:w:q", options, &index)) != -1) {
- switch (opt) {
- case 'u':
- url = optarg;
- break;
- case 'a':
- allocator = optarg;
- break;
- case 'p':
- setenv("LIBPROCESS_PORT", optarg, 1);
- break;
- case 'w':
- webuiPortStr = optarg;
- break;
- case 'q':
- quiet = true;
- break;
- case '?':
- // Error parsing options; getopt prints an error message, so just exit
- exit(1);
- break;
- default:
- break;
- }
+ Params params;
+
+ try {
+ conf.load(argc, argv, true);
+ conf.validate();
+ params = conf.getParams();
+ } catch (BadOptionValueException& e) {
+ cerr << "Invalid value for '" << e.what() << "' option" << endl;
+ exit(1);
+ } catch (ConfigurationException& e) {
+ cerr << "Configuration error: " << e.what() << endl;
+ exit(1);
}
- // TODO(benh): Don't log to /tmp behind a sys admin's back!
- FLAGS_log_dir = "/tmp";
+ if (params.contains("port"))
+ setenv("LIBPROCESS_PORT", params["port"].c_str(), 1);
+
+ FLAGS_log_dir = params["log_dir"];
FLAGS_logbufsecs = 1;
google::InitGoogleLogging(argv[0]);
+ bool quiet = params.get<bool>("quiet", false);
if (!quiet)
google::SetStderrLogging(google::INFO);
+ string url = params.get("url", "");
+
LOG(INFO) << "Build: " << BUILD_DATE << " by " << BUILD_USER;
LOG(INFO) << "Starting Nexus master";
- Master *master = new Master(allocator);
+ if (chdir(dirname(argv[0])) != 0)
+ fatalerror("Could not chdir into %s", dirname(argv[0]));
+
+ Master *master = new Master(params);
PID pid = Process::spawn(master);
MasterDetector *detector = MasterDetector::create(url, pid, true, quiet);
#ifdef NEXUS_WEBUI
- if (chdir(dirname(argv[0])) != 0)
- fatalerror("Could not change into %s for running webui.\n", dirname(argv[0]));
-
- // TODO(*): Since we normally don't use exceptions in Mesos, replace
- // use of an exception here with use of a utility to handle checking
- // that the input string is actually a number that fits
- // in the type being used (in this case, short).
- try {
- lexical_cast<short>(webuiPortStr);
- } catch(bad_lexical_cast &) {
- fatal("Passed invalid string for webui port number.\n");
- }
-
- startMasterWebUI(pid, webuiPortStr);
+ startMasterWebUI(pid);
#endif
Process::wait(pid);
Modified: incubator/mesos/trunk/src/nexus_local.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/nexus_local.cpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/nexus_local.cpp (original)
+++ incubator/mesos/trunk/src/nexus_local.cpp Sun Jun 5 05:43:37 2011
@@ -6,6 +6,7 @@
#include <map>
#include <vector>
+#include "configuration.hpp"
#include "foreach.hpp"
#include "nexus_local.hpp"
#include "process_based_isolation_module.hpp"
@@ -52,7 +53,8 @@ PID launch(int numSlaves, int32_t cpus,
google::SetStderrLogging(google::INFO);
}
- master = new Master();
+ Params conf;
+ master = new Master(conf);
PID pid = Process::spawn(master);
Modified: incubator/mesos/trunk/src/params.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/params.hpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/params.hpp (original)
+++ incubator/mesos/trunk/src/params.hpp Sun Jun 5 05:43:37 2011
@@ -119,6 +119,16 @@ public:
}
template <typename T>
+ T get(const string& key, const T& defaultValue) const
+ {
+ map<string, string>::const_iterator it = params.find(key);
+ if (it != params.end())
+ return lexical_cast<T>(it->second);
+ else
+ return defaultValue;
+ }
+
+ template <typename T>
void set(const string& key, T value)
{
params[key] = lexical_cast<string>(value);
Modified: incubator/mesos/trunk/src/tests/test_configuration.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/tests/test_configuration.cpp?rev=1131876&r1=1131875&r2=1131876&view=diff
==============================================================================
--- incubator/mesos/trunk/src/tests/test_configuration.cpp (original)
+++ incubator/mesos/trunk/src/tests/test_configuration.cpp Sun Jun 5 05:43:37 2011
@@ -45,7 +45,7 @@ TEST(ConfigurationTest, DefaultOptions)
conf.addOption<int>("test1", "Testing option", 500);
conf.addOption<short>("test2", "Another tester", 0);
conf.addOption<long>("test3", "Tests the default\noption.", 2010);
- conf.addOption("test4", "Option without default\noption.");
+ conf.addOption<string>("test4", "Option without default\noption.");
conf.addOption<string>("test5", "Option with a default string.", "arb");
conf.load(ARGC, argv, false);
} );