You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:47:33 UTC
svn commit: r1132125 - in /incubator/mesos/trunk/src: Makefile.in
local/local.cpp local/local.hpp master/master.cpp master/master.hpp
tests/test_master.cpp
Author: benh
Date: Sun Jun 5 08:47:32 2011
New Revision: 1132125
URL: http://svn.apache.org/viewvc?rev=1132125&view=rev
Log:
Made some fixes suggested by Ben. Also replaced the way unit tests get a
deterministic date: there's now a DateUtils class with the currentDate()
method, and it is possible to set a mock date to use through one of its
methods.
Modified:
incubator/mesos/trunk/src/Makefile.in
incubator/mesos/trunk/src/local/local.cpp
incubator/mesos/trunk/src/local/local.hpp
incubator/mesos/trunk/src/master/master.cpp
incubator/mesos/trunk/src/master/master.hpp
incubator/mesos/trunk/src/tests/test_master.cpp
Modified: incubator/mesos/trunk/src/Makefile.in
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/Makefile.in?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/Makefile.in (original)
+++ incubator/mesos/trunk/src/Makefile.in Sun Jun 5 08:47:32 2011
@@ -123,7 +123,7 @@ SWIG_WEBUI_OBJ = $(MASTER_SWIG_WEBUI_OBJ
COMMON_OBJ = common/fatal.o messaging/messages.o common/lock.o \
detector/detector.o common/params.o \
detector/url_processor.o configurator/configurator.o \
- common/string_utils.o common/logging.o
+ common/string_utils.o common/logging.o common/date_utils.o
ifeq ($(WITH_ZOOKEEPER),1)
COMMON_OBJ += detector/zookeeper.o
Modified: incubator/mesos/trunk/src/local/local.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/local/local.cpp?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/local/local.cpp (original)
+++ incubator/mesos/trunk/src/local/local.cpp Sun Jun 5 08:47:32 2011
@@ -58,15 +58,13 @@ PID launch(int numSlaves,
int32_t cpus,
int64_t mem,
bool initLogging,
- bool quiet,
- bool dateInMasterId)
+ bool quiet)
{
Params conf;
conf.set("slaves", numSlaves);
conf.set("cpus", cpus);
conf.set("mem", mem);
conf.set("quiet", quiet);
- conf.set("date_in_master_id", dateInMasterId);
return launch(conf, initLogging);
}
Modified: incubator/mesos/trunk/src/local/local.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/local/local.hpp?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/local/local.hpp (original)
+++ incubator/mesos/trunk/src/local/local.hpp Sun Jun 5 08:47:32 2011
@@ -14,14 +14,12 @@ void registerOptions(Configurator* conf)
// Launch a local cluster with a given number of slaves and given numbers
// of CPUs and memory per slave. Additionally one can also toggle whether
-// to initialize Google Logging, whether to log quietly, and whether to
-// include the date in master IDs (this is useful to disable for unit tests).
+// to initialize Google Logging and whether to log quietly.
PID launch(int numSlaves,
int32_t cpus,
int64_t mem,
bool initLogging,
- bool quiet,
- bool dateInMasterId = true);
+ bool quiet);
// Launch a local cluster with a given configuration.
PID launch(const Params& conf, bool initLogging);
Modified: incubator/mesos/trunk/src/master/master.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master/master.cpp?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master/master.cpp (original)
+++ incubator/mesos/trunk/src/master/master.cpp Sun Jun 5 08:47:32 2011
@@ -2,6 +2,8 @@
#include <glog/logging.h>
+#include "common/date_utils.hpp"
+
#include "allocator.hpp"
#include "allocator_factory.hpp"
#include "master.hpp"
@@ -255,20 +257,14 @@ void Master::operator () ()
{
LOG(INFO) << "Master started at mesos://" << self();
- // Don't do anything until we get a fault tolerance ID.
+ // Don't do anything until we get a master ID.
while (receive() != GOT_MASTER_ID) {
LOG(INFO) << "Oops! We're dropping a message since "
<< "we haven't received an identifier yet!";
}
+ string faultToleranceId;
tie(faultToleranceId) = unpack<GOT_MASTER_ID>(body());
-
- // Create a master ID based on the fault tolerance ID we got.
- // We can optionally exclude the start date to simplify unit tests.
- if (conf.get<bool>("date_in_master_id", true)) {
- masterId = currentDate() + "-" + faultToleranceId;
- } else {
- masterId = faultToleranceId;
- }
+ masterId = DateUtils::currentDate() + "-" + faultToleranceId;
LOG(INFO) << "Master ID: " << masterId;
// Create the allocator (we do this after the constructor because it
@@ -1133,19 +1129,6 @@ FrameworkID Master::newFrameworkId()
}
-// Get the current date in the format used for master IDs (YYYYMMDDhhmm).
-string Master::currentDate()
-{
- time_t rawtime;
- struct tm* timeinfo;
- time(&rawtime);
- timeinfo = localtime(&rawtime);
- char timestr[32];
- strftime(timestr, sizeof(timestr), "%Y%m%d%H%M", timeinfo);
- return timestr;
-}
-
-
const Params& Master::getConf()
{
return conf;
Modified: incubator/mesos/trunk/src/master/master.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/master/master.hpp?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/master/master.hpp (original)
+++ incubator/mesos/trunk/src/master/master.hpp Sun Jun 5 08:47:32 2011
@@ -327,12 +327,9 @@ protected:
string allocatorType;
Allocator *allocator;
- string faultToleranceId; // Differentiates masters in fault tolerant mode;
- // will be this master's ZooKeeper ephemeral id.
-
- string masterId; // Contains the date the master was launched and its
- // faultToleranceId. Used in framework and slave IDs
- // created by this master.
+ string masterId; // Contains the date the master was launched and its fault
+ // tolerance ID (e.g. ephemeral ID returned from ZooKeeper).
+ // Used in framework and slave IDs created by this master.
public:
Master();
Modified: incubator/mesos/trunk/src/tests/test_master.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/src/tests/test_master.cpp?rev=1132125&r1=1132124&r2=1132125&view=diff
==============================================================================
--- incubator/mesos/trunk/src/tests/test_master.cpp (original)
+++ incubator/mesos/trunk/src/tests/test_master.cpp Sun Jun 5 08:47:32 2011
@@ -5,6 +5,8 @@
#include <mesos_exec.hpp>
#include <mesos_sched.hpp>
+#include "common/date_utils.hpp"
+
#include "local/local.hpp"
#include "master/master.hpp"
@@ -75,7 +77,7 @@ public:
TEST(MasterTest, NoopFrameworkWithOneSlave)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 2, 1 * Gigabyte, false, false, false);
+ PID master = local::launch(1, 2, 1 * Gigabyte, false, false);
NoopScheduler sched(1);
MesosSchedulerDriver driver(&sched, master);
driver.run();
@@ -88,7 +90,7 @@ TEST(MasterTest, NoopFrameworkWithOneSla
TEST(MasterTest, NoopFrameworkWithMultipleSlaves)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(10, 2, 1 * Gigabyte, false, false, false);
+ PID master = local::launch(10, 2, 1 * Gigabyte, false, false);
NoopScheduler sched(10);
MesosSchedulerDriver driver(&sched, master);
driver.run();
@@ -132,130 +134,144 @@ public:
TEST(MasterTest, DuplicateTaskIdsInResponse)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "1";
params["mem"] = lexical_cast<string>(1 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
- tasks.push_back(TaskDescription(2, "0-0", "", params, ""));
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
+ tasks.push_back(TaskDescription(2, "200102030405-0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Duplicate task ID: 1", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooMuchMemoryInTask)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "1";
params["mem"] = lexical_cast<string>(4 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Too many resources accepted", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooMuchCpuInTask)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "4";
params["mem"] = lexical_cast<string>(1 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Too many resources accepted", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooLittleCpuInTask)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "0";
params["mem"] = lexical_cast<string>(1 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Invalid task size: <0 CPUs, 1024 MEM>", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooLittleMemoryInTask)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "1";
params["mem"] = "1";
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Invalid task size: <1 CPUs, 1 MEM>", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooMuchMemoryAcrossTasks)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "1";
params["mem"] = lexical_cast<string>(2 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
- tasks.push_back(TaskDescription(2, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
+ tasks.push_back(TaskDescription(2, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Too many resources accepted", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, TooMuchCpuAcrossTasks)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 3, 3 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 3, 3 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "2";
params["mem"] = lexical_cast<string>(1 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
- tasks.push_back(TaskDescription(2, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
+ tasks.push_back(TaskDescription(2, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched(tasks);
MesosSchedulerDriver driver(&sched, master);
driver.run();
EXPECT_EQ("Too many resources accepted", sched.errorMessage);
local::shutdown();
+ DateUtils::clearMockDate();
}
TEST(MasterTest, ResourcesReofferedAfterReject)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(10, 2, 1 * Gigabyte, false, false, false);
+ PID master = local::launch(10, 2, 1 * Gigabyte, false, false);
NoopScheduler sched1(10);
MesosSchedulerDriver driver1(&sched1, master);
@@ -276,13 +292,14 @@ TEST(MasterTest, ResourcesReofferedAfter
TEST(MasterTest, ResourcesReofferedAfterBadResponse)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 2, 1 * Gigabyte, false, false, false);
+ DateUtils::setMockDate("200102030405");
+ PID master = local::launch(1, 2, 1 * Gigabyte, false, false);
vector<TaskDescription> tasks;
map<string, string> params;
params["cpus"] = "0";
params["mem"] = lexical_cast<string>(1 * Gigabyte);
- tasks.push_back(TaskDescription(1, "0-0", "", params, ""));
+ tasks.push_back(TaskDescription(1, "200102030405-0-0", "", params, ""));
FixedResponseScheduler sched1(tasks);
MesosSchedulerDriver driver1(&sched1, master);
driver1.run();
@@ -295,6 +312,7 @@ TEST(MasterTest, ResourcesReofferedAfter
EXPECT_EQ(1, sched2.offersGotten);
local::shutdown();
+ DateUtils::clearMockDate();
}
@@ -414,7 +432,7 @@ TEST(MasterTest, SchedulerFailover)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
- PID master = local::launch(1, 2, 1 * Gigabyte, false, false, false);
+ PID master = local::launch(1, 2, 1 * Gigabyte, false, false);
FailoverScheduler failoverSched;
FailingScheduler failingSched(&failoverSched, master);
@@ -548,7 +566,7 @@ TEST(MasterTest, SlavePartitioned)
ProcessClock::pause();
- PID master = local::launch(1, 2, 1 * Gigabyte, false, false, false);
+ PID master = local::launch(1, 2, 1 * Gigabyte, false, false);
SlavePartitionedScheduler sched;
MesosSchedulerDriver driver(&sched, master);