You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/08/13 21:49:45 UTC
[01/23] git commit: Fixed master to properly do task reconciliation
when slave re-registers.
Updated Branches:
refs/heads/vinod/0.13.0 [created] 2327ae35f
Fixed master to properly do task reconciliation when slave re-registers.
Review: https://reviews.apache.org/r/11229
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/25f136a3
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/25f136a3
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/25f136a3
Branch: refs/heads/vinod/0.13.0
Commit: 25f136a38c77a0cef6cd3b567fa731a68d4a242e
Parents: 22e6450
Author: Vinod Kone <vi...@twitter.com>
Authored: Fri May 17 17:09:36 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
src/master/master.cpp | 91 ++++++++++++++++---------------
src/master/master.hpp | 17 ++++--
src/slave/slave.cpp | 5 ++
src/slave/slave.hpp | 2 -
src/tests/fault_tolerance_tests.cpp | 93 +++++++++++++++++++++++++++++++-
5 files changed, 156 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f136a3/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d031b95..6ac86cd 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -943,50 +943,8 @@ void Master::reregisterSlave(const SlaveID& slaveId,
<< ") is being allowed to re-register with an already"
<< " in use id (" << slaveId << ")";
- // Consolidate tasks between master and the slave.
- // Fist, look for the tasks present in the slave but not present
- // in the master.
- multihashmap<FrameworkID, TaskID> slaveTasks;
- foreach (const Task& task, tasks) {
- if (!slave->tasks.contains(
- std::make_pair(task.framework_id(), task.task_id()))) {
- // This might happen if a terminal status update for this task
- // came before the slave re-registered message.
- // TODO(vinod): Consider sending a KillTaskMessage.
- // TODO(vinod): Export a statistic for these tasks.
- LOG(WARNING) << "Slave " << slaveId << " attempted to re-register"
- << " with unknown task " << task.task_id()
- << " of framework " << task.framework_id();
- }
- slaveTasks.put(task.framework_id(), task.task_id());
- }
-
- // Send TASK_LOST updates for tasks present in the master but
- // missing from the slave. This could happen if the task was
- // dropped by the slave (e.g., slave exited before getting the
- // task or the task was launched while slave was in recovery).
- foreachvalue (Task* task, utils::copy(slave->tasks)) {
- if (!slaveTasks.contains(task->framework_id(), task->task_id())) {
- LOG(WARNING) << "Sending TASK_LOST for task " << task->task_id()
- << " of framework " << task->framework_id()
- << " unknown to the slave " << slaveId;
-
- Framework* framework = getFramework(task->framework_id());
- if (framework != NULL) {
- const StatusUpdate& update = protobuf::createStatusUpdate(
- task->framework_id(),
- slaveId,
- task->task_id(),
- TASK_LOST,
- "Task was not received by the slave");
-
- StatusUpdateMessage message;
- message.mutable_update()->CopyFrom(update);
- send(framework->pid, message);
- }
- removeTask(task);
- }
- }
+ // Reconcile tasks between master and the slave.
+ reconcileTasks(slave, tasks);
SlaveReregisteredMessage message;
message.mutable_slave_id()->MergeFrom(slave->id);
@@ -1646,6 +1604,46 @@ Resources Master::launchTask(const TaskInfo& task,
}
+void Master::reconcileTasks(Slave* slave, const vector<Task>& tasks)
+{
+ CHECK_NOTNULL(slave);
+
+ // We convert the 'tasks' into a map for easier lookup below.
+ // TODO(vinod): Check if the tasks are known to the master.
+ multihashmap<FrameworkID, TaskID> slaveTasks;
+ foreach (const Task& task, tasks) {
+ slaveTasks.put(task.framework_id(), task.task_id());
+ }
+
+ // Send TASK_LOST updates for tasks present in the master but
+ // missing from the slave. This could happen if the task was
+ // dropped by the slave (e.g., slave exited before getting the
+ // task or the task was launched while slave was in recovery).
+ foreachvalue (Task* task, utils::copy(slave->tasks)) {
+ if (!slaveTasks.contains(task->framework_id(), task->task_id())) {
+ LOG(WARNING) << "Sending TASK_LOST for task " << task->task_id()
+ << " of framework " << task->framework_id()
+ << " unknown to the slave " << slave->id;
+
+ Framework* framework = getFramework(task->framework_id());
+ if (framework != NULL) {
+ const StatusUpdate& update = protobuf::createStatusUpdate(
+ task->framework_id(),
+ slave->id,
+ task->task_id(),
+ TASK_LOST,
+ "Task was not received by the slave");
+
+ StatusUpdateMessage message;
+ message.mutable_update()->CopyFrom(update);
+ send(framework->pid, message);
+ }
+ removeTask(task);
+ }
+ }
+}
+
+
void Master::addFramework(Framework* framework, bool reregister)
{
CHECK(frameworks.count(framework->id) == 0);
@@ -1918,6 +1916,11 @@ void Master::readdSlave(Slave* slave,
}
foreach (const Task& task, tasks) {
+ // Ignore tasks that have reached terminal state.
+ if (protobuf::isTerminalState(task.state())) {
+ continue;
+ }
+
Task* t = new Task(task);
// Add the task to the slave.
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f136a3/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 82e3596..f0b1def 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -127,11 +127,18 @@ protected:
// Process a launch tasks request (for a non-cancelled offer) by
// launching the desired tasks (if the offer contains a valid set of
// tasks) and reporting any unused resources to the allocator.
- void processTasks(Offer* offer,
- Framework* framework,
- Slave* slave,
- const std::vector<TaskInfo>& tasks,
- const Filters& filters);
+ void processTasks(
+ Offer* offer,
+ Framework* framework,
+ Slave* slave,
+ const std::vector<TaskInfo>& tasks,
+ const Filters& filters);
+
+ // Reconciles a re-registering slave's tasks and sends TASK_LOST
+ // updates for tasks known to the master but unknown to the slave.
+ void reconcileTasks(
+ Slave* slave,
+ const std::vector<Task>& tasks);
// Add a framework.
void addFramework(Framework* framework, bool reregister = false);
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f136a3/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 840c64d..b5b7e0e 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -703,6 +703,11 @@ void Slave::doReliableRegistration()
message.add_tasks()->CopyFrom(t);
}
+
+ // Add terminated tasks.
+ foreachvalue (Task* task, executor->terminatedTasks) {
+ message.add_tasks()->CopyFrom(*task);
+ }
}
}
send(master, message);
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f136a3/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 5aba7ed..d1ba82e 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -376,8 +376,6 @@ struct Executor
hashmap<TaskID, Task*> terminatedTasks; // Terminated but pending updates.
boost::circular_buffer<Task> completedTasks; // Terminated and updates acked.
- multihashmap<TaskID, UUID> updates; // Pending updates.
-
private:
Executor(const Executor&); // No copying.
Executor& operator = (const Executor&); // No assigning.
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f136a3/src/tests/fault_tolerance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/fault_tolerance_tests.cpp b/src/tests/fault_tolerance_tests.cpp
index ef570b7..4afbbec 100644
--- a/src/tests/fault_tolerance_tests.cpp
+++ b/src/tests/fault_tolerance_tests.cpp
@@ -1378,7 +1378,7 @@ TEST_F(FaultToleranceTest, SlaveReregisterTerminatedExecutor)
// This test verifies that the master sends TASK_LOST updates
// for tasks in the master absent from the re-registered slave.
// We do this by dropping RunTaskMessage from master to the slave.
-TEST_F(FaultToleranceTest, ConsolidateTasksOnSlaveReregistration)
+TEST_F(FaultToleranceTest, ReconcileLostTasks)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
@@ -1448,3 +1448,94 @@ TEST_F(FaultToleranceTest, ConsolidateTasksOnSlaveReregistration)
Shutdown();
}
+
+
+// This test verifies that when the slave re-registers, the master
+// does not send TASK_LOST update for a task that has reached terminal
+// state but is waiting for an acknowledgement.
+TEST_F(FaultToleranceTest, ReconcileIncompleteTasks)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ Try<PID<Slave> > slave = StartSlave(&exec);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(&sched, DEFAULT_FRAMEWORK_INFO, master.get());
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ TaskInfo task;
+ task.set_name("test task");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
+ task.mutable_resources()->MergeFrom(offers.get()[0].resources());
+ task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ EXPECT_CALL(exec, registered(_, _, _, _));
+
+ // Send a terminal update right away.
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED));
+
+ // Drop the status update from slave to the master, so that
+ // the slave has a pending terminal update when it re-registers.
+ DROP_PROTOBUF(StatusUpdateMessage(), _, master.get());
+
+ Future<Nothing> _statusUpdate = FUTURE_DISPATCH(_, &Slave::_statusUpdate);
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ Clock::pause();
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY(_statusUpdate);
+
+ Future<SlaveReregisteredMessage> slaveReregisteredMessage =
+ FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _);
+
+ // Simulate a spurious newMasterDetected event (e.g., due to ZooKeeper
+ // expiration) at the slave to force re-registration.
+ NewMasterDetectedMessage message;
+ message.set_pid(master.get());
+
+ process::post(slave.get(), message);
+
+ AWAIT_READY(slaveReregisteredMessage);
+
+ // The master should not send a TASK_LOST after the slave
+ // re-registers. We check this by advancing the clock so that
+ // the only update the scheduler receives is the retried
+ // TASK_FINISHED update.
+ Clock::advance(STATUS_UPDATE_RETRY_INTERVAL);
+
+ AWAIT_READY(status);
+ ASSERT_EQ(TASK_FINISHED, status.get().state());
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
[06/23] git commit: Fixed slave to not send tasks and executor info
of a terminated executor, when re-registering with master.
Posted by vi...@apache.org.
Fixed slave to not send tasks and executor info of a terminated executor,
when re-registering with master.
Review: https://reviews.apache.org/r/11543
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5912aa2a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5912aa2a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5912aa2a
Branch: refs/heads/vinod/0.13.0
Commit: 5912aa2a127b2af1135396d332388915d7670494
Parents: e029cf5
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed May 29 18:14:01 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 6 +++
src/tests/fault_tolerance_tests.cpp | 78 ++++++++++++++++++++++++++++++++
2 files changed, 84 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5912aa2a/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index e905ab3..8ce1646 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -679,6 +679,12 @@ void Slave::doReliableRegistration()
foreachvalue (Framework* framework, frameworks){
foreachvalue (Executor* executor, framework->executors) {
+ // Ignore terminated executors because they do not consume
+ // any resources.
+ if (executor->state == Executor::TERMINATED) {
+ continue;
+ }
+
// TODO(benh): Kill this once framework_id is required
// on ExecutorInfo.
ExecutorInfo* executorInfo = message.add_executor_infos();
http://git-wip-us.apache.org/repos/asf/mesos/blob/5912aa2a/src/tests/fault_tolerance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/fault_tolerance_tests.cpp b/src/tests/fault_tolerance_tests.cpp
index c8fad7c..ef570b7 100644
--- a/src/tests/fault_tolerance_tests.cpp
+++ b/src/tests/fault_tolerance_tests.cpp
@@ -1297,6 +1297,84 @@ TEST_F(FaultToleranceTest, SlaveReregisterOnZKExpiration)
}
+// This test verifies that a re-registering slave does not inform
+// the master about a terminated executor (and its tasks), when the
+// executor has pending updates. We check this by ensuring that the
+// master sends a TASK_LOST update for the task belonging to the
+// terminated executor.
+TEST_F(FaultToleranceTest, SlaveReregisterTerminatedExecutor)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+ TestingIsolator isolator(&exec);
+
+ Try<PID<Slave> > slave = StartSlave(&isolator);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(&sched, DEFAULT_FRAMEWORK_INFO, master.get());
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(LaunchTasks(1, 1, 512))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ EXPECT_CALL(exec, registered(_, _, _, _));
+
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.start();
+
+ AWAIT_READY(status);
+ EXPECT_EQ(TASK_RUNNING, status.get().state());
+
+ // Drop the TASK_LOST status update(s) sent to the master.
+ // This ensures that the TASK_LOST received by the scheduler
+ // is generated by the master.
+ DROP_PROTOBUFS(StatusUpdateMessage(), _, master.get());
+
+ Future<ExitedExecutorMessage> executorExitedMessage =
+ FUTURE_PROTOBUF(ExitedExecutorMessage(), _, _);
+
+ // Now kill the executor.
+ dispatch(isolator,
+ &Isolator::killExecutor,
+ frameworkId.get(),
+ DEFAULT_EXECUTOR_ID);
+
+ AWAIT_READY(executorExitedMessage);
+
+ // Simulate a spurious newMasterDetected event (e.g., due to ZooKeeper
+ // expiration) at the slave to force re-registration.
+ Future<TaskStatus> status2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status2));
+
+ NewMasterDetectedMessage message;
+ message.set_pid(master.get());
+
+ process::post(slave.get(), message);
+
+ AWAIT_READY(status2);
+ EXPECT_EQ(TASK_LOST, status2.get().state());
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
+
+
// This test verifies that the master sends TASK_LOST updates
// for tasks in the master absent from the re-registered slave.
// We do this by dropping RunTaskMessage from master to the slave.
[15/23] git commit: Fixed slave to validate slave id during
(re-)registration.
Posted by vi...@apache.org.
Fixed slave to validate slave id during (re-)registration.
Review: https://reviews.apache.org/r/12122
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a278acad
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a278acad
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a278acad
Branch: refs/heads/vinod/0.13.0
Commit: a278acaddea1239f4d10c6ff8bdc62745b8972b2
Parents: ad26620
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed Jun 26 14:44:07 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:47:20 2013 -0700
----------------------------------------------------------------------
src/master/master.cpp | 2 +-
src/slave/slave.cpp | 24 +++++++++++++++++-------
2 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a278acad/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d424a69..5782164 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -455,7 +455,7 @@ void Master::exited(const UPID& pid)
// its tasks transitioned to LOST and resources recovered.
foreachvalue (Slave* slave, slaves) {
if (slave->pid == pid) {
- LOG(INFO) << "Slave " << slave->id << "(" << slave->info.hostname()
+ LOG(INFO) << "Slave " << slave->id << " (" << slave->info.hostname()
<< ") disconnected";
// Remove the slave, if it is not checkpointing.
http://git-wip-us.apache.org/repos/asf/mesos/blob/a278acad/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c5380b8..aada544 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -601,7 +601,12 @@ void Slave::registered(const SlaveID& slaveId)
break;
}
case RUNNING:
- // Already registered. Ignore registration.
+ // Already registered!
+ if (!(info.id() == slaveId)) {
+ EXIT(1) << "Registered but got wrong id: " << slaveId
+ << "(expected: " << info.id() << "). Committing suicide";
+ }
+ LOG(WARNING) << "Already registered with master " << master;
break;
case TERMINATING:
LOG(WARNING) << "Ignoring registration because slave is terminating";
@@ -622,12 +627,17 @@ void Slave::reregistered(const SlaveID& slaveId)
state = RUNNING;
if (!(info.id() == slaveId)) {
- LOG(FATAL) << "Slave re-registered but got wrong id: " << slaveId
- << "(expected: " << info.id() << ")";
+ EXIT(1) << "Re-registered but got wrong id: " << slaveId
+ << "(expected: " << info.id() << "). Committing suicide";
}
break;
case RUNNING:
- // Already registered. Ignore registration.
+ // Already re-registered!
+ if (!(info.id() == slaveId)) {
+ EXIT(1) << "Re-registered but got wrong id: " << slaveId
+ << "(expected: " << info.id() << "). Committing suicide";
+ }
+ LOG(WARNING) << "Already re-registered with master " << master;
break;
case TERMINATING:
LOG(WARNING) << "Ignoring re-registration because slave is terminating";
@@ -729,8 +739,8 @@ void Slave::runTask(
// the master when the slave re-registers.
if (!(task.slave_id() == info.id())) {
- LOG(WARNING) << "Ignoring task " << task.task_id()
- << " because it was intended for the old slave " << info.id();
+ LOG(WARNING) << "Slave " << info.id() << " ignoring task " << task.task_id()
+ << " because it was intended for old slave " << task.slave_id();
return;
}
@@ -1675,7 +1685,7 @@ void Slave::reregisterExecutorTimeout()
// 2) When slave generates task updates (e.g LOST/KILLED/FAILED).
void Slave::statusUpdate(const StatusUpdate& update)
{
- LOG(INFO) << "Handling status update " << update;
+ LOG(INFO) << "Handling status update " << update << " from " << from;
CHECK(state == RECOVERING || state == DISCONNECTED ||
state == RUNNING || state == TERMINATING)
[18/23] git commit: Added a comment.
Posted by vi...@apache.org.
Added a comment.
Review: https://reviews.apache.org/r/12527
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5b4151e2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5b4151e2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5b4151e2
Branch: refs/heads/vinod/0.13.0
Commit: 5b4151e2f09bc5b43a8c129063c26498915b8872
Parents: f5a8c59
Author: Benjamin Hindman <be...@gmail.com>
Authored: Fri Jul 12 15:32:46 2013 -0700
Committer: Benjamin Mahler <bm...@twitter.com>
Committed: Fri Jul 12 16:22:04 2013 -0700
----------------------------------------------------------------------
3rdparty/libprocess/include/process/http.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5b4151e2/3rdparty/libprocess/include/process/http.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/http.hpp b/3rdparty/libprocess/include/process/http.hpp
index 751cfb8..d0a26d3 100644
--- a/3rdparty/libprocess/include/process/http.hpp
+++ b/3rdparty/libprocess/include/process/http.hpp
@@ -33,8 +33,8 @@ struct Request
// Tracked by: https://issues.apache.org/jira/browse/MESOS-328.
hashmap<std::string, std::string> headers;
std::string method;
+ std::string url; // path/query#fragment
std::string path;
- std::string url;
std::string fragment;
hashmap<std::string, std::string> query;
std::string body;
[19/23] git commit: Added a '/__processes' endpoint.
Posted by vi...@apache.org.
Added a '/__processes' endpoint.
Review: https://reviews.apache.org/r/12528
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d6e399aa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d6e399aa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d6e399aa
Branch: refs/heads/vinod/0.13.0
Commit: d6e399aa827f990e3a4b228e561f15d5c35c4fb7
Parents: 5b4151e
Author: Benjamin Hindman <be...@gmail.com>
Authored: Fri Jul 12 15:33:25 2013 -0700
Committer: Benjamin Mahler <bm...@twitter.com>
Committed: Fri Jul 12 16:22:15 2013 -0700
----------------------------------------------------------------------
3rdparty/libprocess/src/process.cpp | 142 ++++++++++++++++++++++++++++++-
1 file changed, 141 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d6e399aa/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index 3ffe0b5..373097c 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -295,6 +295,56 @@ private:
};
+// Helper for creating routes without a process.
+// TODO(benh): Move this into route.hpp.
+class Route
+{
+public:
+ Route(const string& name,
+ const Option<string>& help,
+ const lambda::function<Future<Response>(const Request&)>& handler)
+ {
+ process = new RouteProcess(name, help, handler);
+ spawn(process);
+ }
+
+ ~Route()
+ {
+ terminate(process);
+ wait(process);
+ }
+
+private:
+ class RouteProcess : public Process<RouteProcess>
+ {
+ public:
+ RouteProcess(
+ const string& name,
+ const Option<string>& _help,
+ const lambda::function<Future<Response>(const Request&)>& _handler)
+ : ProcessBase(strings::remove(name, "/", strings::PREFIX)),
+ help(_help),
+ handler(_handler) {}
+
+ protected:
+ virtual void initialize()
+ {
+ route("/", help, &RouteProcess::handle);
+ }
+
+ Future<Response> handle(const Request& request)
+ {
+ return handler(request);
+ }
+
+ const Option<string> help;
+ const lambda::function<Future<Response>(const Request&)> handler;
+ };
+
+ RouteProcess* process;
+};
+
+
class SocketManager
{
public:
@@ -391,6 +441,9 @@ public:
void settle();
+ // The /__processes__ route.
+ Future<Response> __processes__(const Request&);
+
private:
// Delegate process name to receive root HTTP requests.
const string delegate;
@@ -1410,7 +1463,7 @@ void initialize(const string& delegate)
// TODO(bmahler): Investigate memory implications of this window
// size. We may also want to provide a maximum memory size rather than
// time window. Or, offload older data to disk, etc.
- process::statistics = new Statistics(Weeks(2));
+ statistics = new Statistics(Weeks(2));
// Initialize the mime types.
mime::initialize();
@@ -1418,6 +1471,12 @@ void initialize(const string& delegate)
// Initialize the response statuses.
http::initialize();
+ // Add a route for getting process information.
+ lambda::function<Future<Response>(const Request&)> __processes__ =
+ lambda::bind(&ProcessManager::__processes__, process_manager, lambda::_1);
+
+ new Route("/__processes__", None(), __processes__);
+
char temp[INET_ADDRSTRLEN];
if (inet_ntop(AF_INET, (in_addr*) &__ip__, temp, INET_ADDRSTRLEN) == NULL) {
PLOG(FATAL) << "Failed to initialize, inet_ntop";
@@ -2313,6 +2372,7 @@ bool ProcessManager::deliver(
return true;
}
+
bool ProcessManager::deliver(
const UPID& to,
Event* event,
@@ -2769,6 +2829,86 @@ void ProcessManager::settle()
}
+Future<Response> ProcessManager::__processes__(const Request&)
+{
+ JSON::Array array;
+
+ synchronized (processes) {
+ foreachvalue (const ProcessBase* process, process_manager->processes) {
+ JSON::Object object;
+ object.values["id"] = process->pid.id;
+
+ JSON::Array events;
+
+ struct JSONVisitor : EventVisitor
+ {
+ JSONVisitor(JSON::Array* _events) : events(_events) {}
+
+ virtual void visit(const MessageEvent& event)
+ {
+ JSON::Object object;
+ object.values["type"] = "MESSAGE";
+
+ const Message& message = *event.message;
+
+ object.values["name"] = message.name;
+ object.values["from"] = string(message.from);
+ object.values["to"] = string(message.to);
+ object.values["body"] = message.body;
+
+ events->values.push_back(object);
+ }
+
+ virtual void visit(const HttpEvent& event)
+ {
+ JSON::Object object;
+ object.values["type"] = "HTTP";
+
+ const Request& request = *event.request;
+
+ object.values["method"] = request.method;
+ object.values["url"] = request.url;
+
+ events->values.push_back(object);
+ }
+
+ virtual void visit(const DispatchEvent& event)
+ {
+ JSON::Object object;
+ object.values["type"] = "DISPATCH";
+ events->values.push_back(object);
+ }
+
+ virtual void visit(const ExitedEvent& event)
+ {
+ JSON::Object object;
+ object.values["type"] = "EXITED";
+ events->values.push_back(object);
+ }
+
+ virtual void visit(const TerminateEvent& event)
+ {
+ JSON::Object object;
+ object.values["type"] = "TERMINATE";
+ events->values.push_back(object);
+ }
+
+ JSON::Array* events;
+ } visitor(&events);
+
+ foreach (Event* event, process->events) {
+ event->visit(&visitor);
+ }
+
+ object.values["events"] = events;
+ array.values.push_back(object);
+ }
+ }
+
+ return OK(array);
+}
+
+
Timer Timer::create(
const Duration& duration,
const lambda::function<void(void)>& thunk)
[14/23] git commit: Improved the configure output when deprecated
headers are present on OS X.
Posted by vi...@apache.org.
Improved the configure output when deprecated headers are present
on OS X.
Review: https://reviews.apache.org/r/12045
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ad266206
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ad266206
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ad266206
Branch: refs/heads/vinod/0.13.0
Commit: ad2662067e3bfc8fd7d1d7eaaf4b30c2af3ed697
Parents: 8b6292d
Author: Benjamin Mahler <bm...@twitter.com>
Authored: Fri Jun 21 17:00:44 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:47:20 2013 -0700
----------------------------------------------------------------------
configure.ac | 15 +++++++++++++--
src/master/master.cpp | 3 ++-
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ad266206/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index e76f775..6b9a29d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -318,7 +318,17 @@ __EOF__
["-I/System/Library/Frameworks/JavaVM.framework/Headers"]],
[JAVA_CPPFLAGS=java_cppflags
TRY_LINK_JNI([break])])
- AC_MSG_ERROR([failed to build with JNI]) # Exhausted options.
+ # Exhausted options.
+ AC_MSG_ERROR([failed to build with JNI
+ -------------------------------------------------------------------
+ It appears we were unable to compile against the JNI. This is most
+ likely due to one of the following issues:
+ 1. You do not have a JDK installed on your system.
+ 2. All JDKs installed on your system have deprecated JNI headers.
+ It is advised to install OpenJDK on your system, as the JDK that
+ ships with OS X has deprecated JNI headers.
+ -------------------------------------------------------------------
+ ])
done
else
while true; do # Loop until sucessful (via break) or exhausted options.
@@ -326,7 +336,8 @@ __EOF__
[["-I$JAVA_HOME/include -I$JAVA_HOME/include/$OS_NAME"]],
[JAVA_CPPFLAGS=java_cppflags
TRY_LINK_JNI([break])])
- AC_MSG_ERROR([failed to build with JNI]) # Exhausted options.
+ # Exhausted options.
+ AC_MSG_ERROR([failed to build with JNI])
done
fi
else
http://git-wip-us.apache.org/repos/asf/mesos/blob/ad266206/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index df5794b..d424a69 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1981,7 +1981,8 @@ void Master::removeSlave(Slave* slave)
task->task_id(),
TASK_LOST,
"Slave " + slave->info.hostname() + " removed",
- (task->has_executor_id() ? task->executor_id() : None())));
+ (task->has_executor_id() ?
+ Option<ExecutorID>(task->executor_id()) : None())));
LOG(INFO) << "Sending status update " << message.update()
<< " due to the removal of slave "
[05/23] git commit: Fixed Zookeeper to recursively create parent
paths as necessary.
Posted by vi...@apache.org.
Fixed Zookeeper to recursively create parent paths as necessary.
Review: https://reviews.apache.org/r/11366
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9029967d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9029967d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9029967d
Branch: refs/heads/vinod/0.13.0
Commit: 9029967d96c0f2cc09a2b09ab3d0a8d0ab4005b7
Parents: cc5e1e7
Author: Vinod Kone <vi...@twitter.com>
Authored: Thu May 23 23:26:31 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
src/tests/zookeeper_tests.cpp | 12 ++++-----
src/zookeeper/zookeeper.cpp | 51 +++++++++++++++++++-------------------
src/zookeeper/zookeeper.hpp | 2 +-
3 files changed, 33 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9029967d/src/tests/zookeeper_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/zookeeper_tests.cpp b/src/tests/zookeeper_tests.cpp
index 77a5ab2..16c5fb7 100644
--- a/src/tests/zookeeper_tests.cpp
+++ b/src/tests/zookeeper_tests.cpp
@@ -82,12 +82,12 @@ TEST_F(ZooKeeperTest, Create)
ZooKeeper nonOwnerZk(server->connectString(), NO_TIMEOUT, &watcher);
watcher.awaitSessionEvent(ZOO_CONNECTED_STATE);
nonOwnerZk.authenticate("digest", "non-owner:non-owner");
- EXPECT_EQ(ZNOAUTH, nonOwnerZk.create("/foo/bar/baz",
- "",
- zookeeper::EVERYONE_READ_CREATOR_ALL,
- 0,
- NULL,
- true));
+ EXPECT_EQ(ZNODEEXISTS, nonOwnerZk.create("/foo/bar/baz",
+ "",
+ zookeeper::EVERYONE_READ_CREATOR_ALL,
+ 0,
+ NULL,
+ true));
EXPECT_EQ(ZOK, nonOwnerZk.create("/foo/bar/baz/bam",
"44",
zookeeper::EVERYONE_READ_CREATOR_ALL,
http://git-wip-us.apache.org/repos/asf/mesos/blob/9029967d/src/zookeeper/zookeeper.cpp
----------------------------------------------------------------------
diff --git a/src/zookeeper/zookeeper.cpp b/src/zookeeper/zookeeper.cpp
index 267c38a..de33003 100644
--- a/src/zookeeper/zookeeper.cpp
+++ b/src/zookeeper/zookeeper.cpp
@@ -517,41 +517,42 @@ int ZooKeeper::authenticate(const string& scheme, const string& credentials)
}
-int ZooKeeper::create(const string& path, const string& data,
- const ACL_vector& acl, int flags, string* result,
- bool recursive)
+int ZooKeeper::create(
+ const string& path,
+ const string& data,
+ const ACL_vector& acl,
+ int flags,
+ string* result,
+ bool recursive)
{
if (!recursive) {
return impl->create(path, data, acl, flags, result).get();
}
- // Do "recursive" create, i.e., ensure intermediate znodes exist.
- string prefix = "/";
- int code = ZOK;
- foreach (const string& token, strings::tokenize(path, "/")) {
- prefix = path::join(prefix, token);
-
- // Make sure we include 'flags' and 'data' for the final znode.
- if (prefix == path || (prefix + "/") == path) {
- code = impl->create(path, data, acl, flags, result).get();
- } else {
- code = impl->create(prefix, "", acl, 0, result).get();
- }
+ // First check if the path exists.
+ int code = impl->exists(path, false, NULL).get();
+ if (code == ZOK) {
+ return ZNODEEXISTS;
+ }
- // We fail all non-OK return codes except for:
- // ZNODEEXISTS says the node in the znode path we are trying to
- // create already exists - this is what we wanted, so we
- // continue.
- // ZNOAUTH says we can't write the node, but it doesn't tell us
- // whether the node already exists. We take the optimistic
- // approach and assume the node's parent doesn't allow us to
- // write an already existing node (but it exists).
- if (code != ZOK && code != ZNODEEXISTS && code != ZNOAUTH) {
+ // Now recursively create the parent path.
+ // NOTE: We don't use 'dirname()' to get the parent path here
+ // because, it doesn't return the expected path when a path ends
+ // with "/". For example, to create path "/a/b/", we want to
+ // recursively create "/a/b", instead of just creating "/a".
+ const string& parent = path.substr(0, path.find_last_of("/"));
+ if (!parent.empty()) {
+ code = create(parent, "", acl, 0, result, true);
+ if (code != ZOK && code != ZNODEEXISTS) {
return code;
}
}
- return code;
+ // Finally create the path.
+ // TODO(vinod): Delete any intermediate nodes created if this fails.
+ // This requires synchronization because the deletion might affect
+ // other callers (different threads/processes) acting on this path.
+ return impl->create(path, data, acl, flags, result).get();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/9029967d/src/zookeeper/zookeeper.hpp
----------------------------------------------------------------------
diff --git a/src/zookeeper/zookeeper.hpp b/src/zookeeper/zookeeper.hpp
index 99e689e..7243543 100644
--- a/src/zookeeper/zookeeper.hpp
+++ b/src/zookeeper/zookeeper.hpp
@@ -184,7 +184,7 @@ public:
const ACL_vector &acl,
int flags,
std::string *result,
- bool recursive = false);
+ bool recursive = false);
/**
* \brief delete a node in zookeeper synchronously.
[23/23] git commit: Revert "Fixed master to send a
FrameworkReregistered message when the framework re-registers with a failed
over master."
Posted by vi...@apache.org.
Revert "Fixed master to send a FrameworkReregistered message when the
framework re-registers with a failed over master."
This reverts commit 68ccbf1d9ab1fee2bef933a27368e5460f9be42e.
Conflicts:
src/master/master.cpp
src/tests/fault_tolerance_tests.cpp
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2327ae35
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2327ae35
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2327ae35
Branch: refs/heads/vinod/0.13.0
Commit: 2327ae35f23d641e2746bf2fc55d1a0f58ae09af
Parents: 88b589c
Author: Vinod Kone <vi...@twitter.com>
Authored: Mon Aug 12 23:53:18 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Aug 13 12:48:29 2013 -0700
----------------------------------------------------------------------
src/master/master.cpp | 19 ++++++-------------
src/master/master.hpp | 2 +-
src/tests/allocator_zookeeper_tests.cpp | 4 ++--
src/tests/fault_tolerance_tests.cpp | 14 ++++++++------
4 files changed, 17 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2327ae35/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index e9f2225..72a499a 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -678,7 +678,7 @@ void Master::reregisterFramework(const FrameworkInfo& frameworkInfo,
// N.B. Need to add the framwwork _after_ we add it's tasks
// (above) so that we can properly determine the resources it's
// currently using!
- addFramework(framework, true);
+ addFramework(framework);
}
CHECK(frameworks.count(frameworkInfo.id()) > 0);
@@ -1650,7 +1650,7 @@ void Master::reconcileTasks(Slave* slave, const vector<Task>& tasks)
}
-void Master::addFramework(Framework* framework, bool reregister)
+void Master::addFramework(Framework* framework)
{
CHECK(frameworks.count(framework->id) == 0);
@@ -1658,17 +1658,10 @@ void Master::addFramework(Framework* framework, bool reregister)
link(framework->pid);
- if (reregister) {
- FrameworkReregisteredMessage message;
- message.mutable_framework_id()->MergeFrom(framework->id);
- message.mutable_master_info()->MergeFrom(info);
- send(framework->pid, message);
- } else {
- FrameworkRegisteredMessage message;
- message.mutable_framework_id()->MergeFrom(framework->id);
- message.mutable_master_info()->MergeFrom(info);
- send(framework->pid, message);
- }
+ FrameworkRegisteredMessage message;
+ message.mutable_framework_id()->MergeFrom(framework->id);
+ message.mutable_master_info()->MergeFrom(info);
+ send(framework->pid, message);
allocator->frameworkAdded(
framework->id, framework->info, framework->resources);
http://git-wip-us.apache.org/repos/asf/mesos/blob/2327ae35/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index f0b1def..2d8a278 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -141,7 +141,7 @@ protected:
const std::vector<Task>& tasks);
// Add a framework.
- void addFramework(Framework* framework, bool reregister = false);
+ void addFramework(Framework* framework);
// Replace the scheduler for a framework with a new process ID, in
// the event of a scheduler failover.
http://git-wip-us.apache.org/repos/asf/mesos/blob/2327ae35/src/tests/allocator_zookeeper_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/allocator_zookeeper_tests.cpp b/src/tests/allocator_zookeeper_tests.cpp
index 1034d72..e517f50 100644
--- a/src/tests/allocator_zookeeper_tests.cpp
+++ b/src/tests/allocator_zookeeper_tests.cpp
@@ -179,7 +179,7 @@ TYPED_TEST(AllocatorZooKeeperTest, FrameworkReregistersFirst)
.WillOnce(DoAll(InvokeFrameworkAdded(&allocator2),
FutureSatisfy(&frameworkAdded)));
- EXPECT_CALL(sched, reregistered(&driver, _));
+ EXPECT_CALL(sched, registered(&driver, _, _));
AWAIT_READY(frameworkAdded);
@@ -301,7 +301,7 @@ TYPED_TEST(AllocatorZooKeeperTest, SlaveReregistersFirst)
.WillOnce(DoAll(InvokeSlaveAdded(&allocator2),
FutureSatisfy(&slaveAdded)));
- EXPECT_CALL(sched, reregistered(&driver, _));
+ EXPECT_CALL(sched, registered(&driver, _, _));
AWAIT_READY(slaveAdded);
http://git-wip-us.apache.org/repos/asf/mesos/blob/2327ae35/src/tests/fault_tolerance_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/fault_tolerance_tests.cpp b/src/tests/fault_tolerance_tests.cpp
index 4afbbec..cdf4d66 100644
--- a/src/tests/fault_tolerance_tests.cpp
+++ b/src/tests/fault_tolerance_tests.cpp
@@ -594,7 +594,9 @@ TEST_F(FaultToleranceTest, PartitionedSlaveExitedExecutor)
// This test ensures that a framework connecting with a
-// failed over master gets a re-registered callback.
+// failed over master gets a registered callback.
+// Note that this behavior might change in the future and
+// the scheduler might receive a re-registered callback.
TEST_F(FaultToleranceTest, MasterFailover)
{
Try<PID<Master> > master = StartMaster();
@@ -619,9 +621,9 @@ TEST_F(FaultToleranceTest, MasterFailover)
EXPECT_CALL(sched, disconnected(&driver));
- Future<Nothing> reregistered;
- EXPECT_CALL(sched, reregistered(&driver, _))
- .WillOnce(FutureSatisfy(&reregistered));
+ Future<Nothing> registered;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureSatisfy(®istered));
// Simulate a new master detected message to the scheduler.
NewMasterDetectedMessage newMasterDetectedMsg;
@@ -629,8 +631,8 @@ TEST_F(FaultToleranceTest, MasterFailover)
process::post(frameworkRegisteredMessage.get().to, newMasterDetectedMsg);
- // Framework should get a re-register callback.
- AWAIT_READY(reregistered);
+ // Framework should get a registered callback.
+ AWAIT_READY(registered);
driver.stop();
driver.join();
[11/23] git commit: Fixed slave to create work directory during
initialization.
Posted by vi...@apache.org.
Fixed slave to create work directory during initialization.
Review: https://reviews.apache.org/r/12010
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7c306d77
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7c306d77
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7c306d77
Branch: refs/heads/vinod/0.13.0
Commit: 7c306d77a24fcbaf201f20267a9019fc0fc9dd29
Parents: 09f4c11
Author: Vinod Kone <vi...@twitter.com>
Authored: Thu Jun 20 10:24:24 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:17:50 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 68 ++++++++++++++++++++----------------------------
src/slave/slave.hpp | 5 ----
2 files changed, 28 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7c306d77/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index fd96de9..c5380b8 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -76,15 +76,14 @@ namespace slave {
using namespace state;
-Slave::Slave(const Resources& _resources,
+Slave::Slave(const slave::Flags& _flags,
bool _local,
Isolator* _isolator,
Files* _files)
: ProcessBase(ID::generate("slave")),
state(RECOVERING),
- flags(),
+ flags(_flags),
local(_local),
- resources(_resources),
completedFrameworks(MAX_COMPLETED_FRAMEWORKS),
isolator(_isolator),
files(_files),
@@ -93,23 +92,32 @@ Slave::Slave(const Resources& _resources,
metaDir(paths::getMetaRootDir(flags.work_dir)) {}
-Slave::Slave(const slave::Flags& _flags,
- bool _local,
- Isolator* _isolator,
- Files* _files)
- : ProcessBase(ID::generate("slave")),
- state(RECOVERING),
- flags(_flags),
- local(_local),
- completedFrameworks(MAX_COMPLETED_FRAMEWORKS),
- isolator(_isolator),
- files(_files),
- monitor(_isolator),
- statusUpdateManager(new StatusUpdateManager()),
- metaDir(paths::getMetaRootDir(flags.work_dir))
+Slave::~Slave()
{
- // TODO(benh): Move this computation into Flags as the "default".
+ // TODO(benh): Shut down frameworks?
+
+ // TODO(benh): Shut down executors? The executor should get an "exited"
+ // event and initiate a shut down itself.
+ foreachvalue (Framework* framework, frameworks) {
+ delete framework;
+ }
+
+ delete statusUpdateManager;
+}
+
+
+void Slave::initialize()
+{
+ LOG(INFO) << "Slave started on " << string(self()).substr(6);
+
+ // Ensure slave work directory exists.
+ CHECK_SOME(os::mkdir(flags.work_dir))
+ << "Failed to create slave work directory '" << flags.work_dir << "'";
+
+ // Properly set up resources.
+ // TODO(benh): Move this computation into Flags as the "default".
+ // TODO(vinod): Move some of this computation into Resources.
resources = Resources::parse(
flags.resources.isSome() ? flags.resources.get() : "");
@@ -190,31 +198,11 @@ Slave::Slave(const slave::Flags& _flags,
resources = Resources::parse(defaults.get());
+ LOG(INFO) << "Slave resources: " << resources;
+
if (flags.attributes.isSome()) {
attributes = Attributes::parse(flags.attributes.get());
}
-}
-
-
-Slave::~Slave()
-{
- // TODO(benh): Shut down frameworks?
-
- // TODO(benh): Shut down executors? The executor should get an "exited"
- // event and initiate a shut down itself.
-
- foreachvalue (Framework* framework, frameworks) {
- delete framework;
- }
-
- delete statusUpdateManager;
-}
-
-
-void Slave::initialize()
-{
- LOG(INFO) << "Slave started on " << string(self()).substr(6);
- LOG(INFO) << "Slave resources: " << resources;
// Determine our hostname.
Try<string> result = os::hostname();
http://git-wip-us.apache.org/repos/asf/mesos/blob/7c306d77/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index d1ba82e..3a147f0 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -74,11 +74,6 @@ struct Framework;
class Slave : public ProtobufProcess<Slave>
{
public:
- Slave(const Resources& resources,
- bool local,
- Isolator* isolator,
- Files* files);
-
Slave(const Flags& flags,
bool local,
Isolator *isolator,
[12/23] git commit: Added a log message in the master for tasks lost
during slave removal.
Posted by vi...@apache.org.
Added a log message in the master for tasks lost during slave removal.
Review: https://reviews.apache.org/r/12044
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8b6292dc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8b6292dc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8b6292dc
Branch: refs/heads/vinod/0.13.0
Commit: 8b6292dcc4fcacc00c2253a90f3e6dbb3170c028
Parents: 7c306d7
Author: Benjamin Mahler <bm...@twitter.com>
Authored: Fri Jun 21 16:37:10 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:18:08 2013 -0700
----------------------------------------------------------------------
src/master/master.cpp | 25 ++++++++++++-------------
1 file changed, 12 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8b6292dc/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 6ac86cd..df5794b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -1974,20 +1974,19 @@ void Master::removeSlave(Slave* slave)
// Master::reregisterSlave.
if (framework != NULL) {
StatusUpdateMessage message;
- StatusUpdate* update = message.mutable_update();
- update->mutable_framework_id()->MergeFrom(task->framework_id());
-
- if (task->has_executor_id()) {
- update->mutable_executor_id()->MergeFrom(task->executor_id());
- }
+ message.mutable_update()->CopyFrom(
+ protobuf::createStatusUpdate(
+ task->framework_id(),
+ task->slave_id(),
+ task->task_id(),
+ TASK_LOST,
+ "Slave " + slave->info.hostname() + " removed",
+ (task->has_executor_id() ? task->executor_id() : None())));
+
+ LOG(INFO) << "Sending status update " << message.update()
+ << " due to the removal of slave "
+ << slave->id << " (" << slave->info.hostname() << ")";
- update->mutable_slave_id()->MergeFrom(task->slave_id());
- TaskStatus* status = update->mutable_status();
- status->mutable_task_id()->MergeFrom(task->task_id());
- status->set_state(TASK_LOST);
- status->set_message("Slave " + slave->info.hostname() + " removed");
- update->set_timestamp(Clock::now().secs());
- update->set_uuid(UUID::random().toBytes());
send(framework->pid, message);
}
removeTask(task);
[17/23] git commit: Added timing information to Scheduler and
Executor callbacks when verbose logging is enabled.
Posted by vi...@apache.org.
Added timing information to Scheduler and Executor callbacks when
verbose logging is enabled.
Review: https://reviews.apache.org/r/12520
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f5a8c59e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f5a8c59e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f5a8c59e
Branch: refs/heads/vinod/0.13.0
Commit: f5a8c59edf9914adfee180ac8397452da667704a
Parents: a6a2fe9
Author: Benjamin Mahler <bm...@twitter.com>
Authored: Fri Jul 12 13:57:34 2013 -0700
Committer: Benjamin Mahler <bm...@twitter.com>
Committed: Fri Jul 12 16:21:45 2013 -0700
----------------------------------------------------------------------
src/exec/exec.cpp | 60 ++++++++++++++++++++++++++++++++++++++++
src/sched/sched.cpp | 71 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 131 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f5a8c59e/src/exec/exec.cpp
----------------------------------------------------------------------
diff --git a/src/exec/exec.cpp b/src/exec/exec.cpp
index 116168f..d467724 100644
--- a/src/exec/exec.cpp
+++ b/src/exec/exec.cpp
@@ -39,6 +39,7 @@
#include <stout/numify.hpp>
#include <stout/option.hpp>
#include <stout/os.hpp>
+#include <stout/stopwatch.hpp>
#include <stout/stringify.hpp>
#include <stout/uuid.hpp>
@@ -194,7 +195,15 @@ protected:
VLOG(1) << "Executor registered on slave " << slaveId;
connected = true;
+
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->registered(driver, executorInfo, frameworkInfo, slaveInfo);
+
+ VLOG(1) << "Executor::registered took " << stopwatch.elapsed();
}
void reregistered(const SlaveID& slaveId, const SlaveInfo& slaveInfo)
@@ -207,7 +216,14 @@ protected:
VLOG(1) << "Executor re-registered on slave " << slaveId;
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->reregistered(driver, slaveInfo);
+
+ VLOG(1) << "Executor::reregistered took " << stopwatch.elapsed();
}
void reconnect(const SlaveID& slaveId)
@@ -257,7 +273,14 @@ protected:
VLOG(1) << "Executor asked to run task '" << task.task_id() << "'";
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->launchTask(driver, task);
+
+ VLOG(1) << "Executor::launchTask took " << stopwatch.elapsed();
}
void killTask(const TaskID& taskId)
@@ -270,7 +293,14 @@ protected:
VLOG(1) << "Executor asked to kill task '" << taskId << "'";
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->killTask(driver, taskId);
+
+ VLOG(1) << "Executor::killTask took " << stopwatch.elapsed();
}
void statusUpdateAcknowledgement(
@@ -310,7 +340,14 @@ protected:
VLOG(1) << "Executor received framework message";
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->frameworkMessage(driver, data);
+
+ VLOG(1) << "Executor::frameworkMessage took " << stopwatch.elapsed();
}
void shutdown()
@@ -327,8 +364,16 @@ protected:
spawn(new ShutdownProcess(), true);
}
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
// TODO(benh): Any need to invoke driver.stop?
executor->shutdown(driver);
+
+ VLOG(1) << "Executor::shutdown took " << stopwatch.elapsed();
+
aborted = true; // To make sure not to accept any new messages.
if (local) {
@@ -365,8 +410,16 @@ protected:
spawn(new ShutdownProcess(), true);
}
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
// TODO: Pass an argument to shutdown to tell it this is abnormal?
executor->shutdown(driver);
+
+ VLOG(1) << "Executor::shutdown took " << stopwatch.elapsed();
+
aborted = true; // To make sure not to accept any new messages.
// This is a pretty bad state ... no slave is left. Rather
@@ -387,8 +440,15 @@ protected:
driver->abort();
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
executor->error(driver, "Attempted to send TASK_STAGING status update");
+ VLOG(1) << "Executor::error took " << stopwatch.elapsed();
+
return;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/f5a8c59e/src/sched/sched.cpp
----------------------------------------------------------------------
diff --git a/src/sched/sched.cpp b/src/sched/sched.cpp
index 248e12e..8e60336 100644
--- a/src/sched/sched.cpp
+++ b/src/sched/sched.cpp
@@ -43,6 +43,7 @@
#include <stout/flags.hpp>
#include <stout/hashmap.hpp>
#include <stout/os.hpp>
+#include <stout/stopwatch.hpp>
#include <stout/uuid.hpp>
#include "common/lock.hpp"
@@ -182,7 +183,14 @@ protected:
// If master failed over, inform the scheduler about the
// disconnection.
if (connected) {
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->disconnected(driver);
+
+ VLOG(1) << "Scheduler::disconnected took " << stopwatch.elapsed();
}
connected = false;
@@ -196,7 +204,14 @@ protected:
// Inform the scheduler about the disconnection if the driver
// was previously registered with the master.
if (connected) {
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->disconnected(driver);
+
+ VLOG(1) << "Scheduler::disconnected took " << stopwatch.elapsed();
}
// In this case, we don't actually invoke Scheduler::error
@@ -226,7 +241,14 @@ protected:
connected = true;
failover = false;
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->registered(driver, frameworkId, masterInfo);
+
+ VLOG(1) << "Scheduler::registered took " << stopwatch.elapsed();
}
void reregistered(const FrameworkID& frameworkId, const MasterInfo& masterInfo)
@@ -250,7 +272,14 @@ protected:
connected = true;
failover = false;
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->reregistered(driver, masterInfo);
+
+ VLOG(1) << "Scheduler::reregistered took " << stopwatch.elapsed();
}
void doReliableRegistration()
@@ -301,7 +330,14 @@ protected:
}
}
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->resourceOffers(driver, offers);
+
+ VLOG(1) << "Scheduler::resourceOffers took " << stopwatch.elapsed();
}
void rescindOffer(const OfferID& offerId)
@@ -316,7 +352,14 @@ protected:
savedOffers.erase(offerId);
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->offerRescinded(driver, offerId);
+
+ VLOG(1) << "Scheduler::offerRescinded took " << stopwatch.elapsed();
}
void statusUpdate(const StatusUpdate& update, const UPID& pid)
@@ -342,8 +385,15 @@ protected:
// multiple times (of course, if a scheduler re-uses a TaskID,
// that could be bad.
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->statusUpdate(driver, status);
+ VLOG(1) << "Scheduler::statusUpdate took " << stopwatch.elapsed();
+
// Acknowledge the status update.
// NOTE: We do a dispatch here instead of directly sending the ACK because,
// we want to avoid sending the ACK if the driver was aborted when we
@@ -390,7 +440,14 @@ protected:
savedSlavePids.erase(slaveId);
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->slaveLost(driver, slaveId);
+
+ VLOG(1) << "Scheduler::slaveLost took " << stopwatch.elapsed();
}
void frameworkMessage(const SlaveID& slaveId,
@@ -405,7 +462,14 @@ protected:
VLOG(1) << "Received framework message";
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->frameworkMessage(driver, executorId, slaveId, data);
+
+ VLOG(1) << "Scheduler::frameworkMessage took " << stopwatch.elapsed();
}
void error(const string& message)
@@ -419,7 +483,14 @@ protected:
driver->abort();
+ Stopwatch stopwatch;
+ if (FLAGS_v >= 1) {
+ stopwatch.start();
+ }
+
scheduler->error(driver, message);
+
+ VLOG(1) << "Scheduler::error took " << stopwatch.elapsed();
}
void stop(bool failover)
[04/23] git commit: Exposed version in "/vars" and "/state.json"
endpoints.
Posted by vi...@apache.org.
Exposed version in "/vars" and "/state.json" endpoints.
Review: https://reviews.apache.org/r/11642
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e029cf50
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e029cf50
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e029cf50
Branch: refs/heads/vinod/0.13.0
Commit: e029cf507fa970b331bb95f6e2946e0e1b676997
Parents: 9029967
Author: Vinod Kone <vi...@twitter.com>
Authored: Tue Jun 4 15:05:42 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
src/master/http.cpp | 10 +++++++---
src/slave/http.cpp | 10 +++++++---
2 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/e029cf50/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 7e105a4..47471d6 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -22,6 +22,8 @@
#include <string>
#include <vector>
+#include <mesos/mesos.hpp>
+
#include <stout/foreach.hpp>
#include <stout/json.hpp>
#include <stout/net.hpp>
@@ -225,9 +227,10 @@ Future<Response> vars(
std::ostringstream out;
out <<
- "build_date " << build::DATE << "\n" <<
- "build_user " << build::USER << "\n" <<
- "build_flags " << build::FLAGS << "\n";
+ "version: " << MESOS_VERSION << "\n" <<
+ "build_date: " << build::DATE << "\n" <<
+ "build_user: " << build::USER << "\n" <<
+ "build_flags: " << build::FLAGS << "\n";
// TODO(benh): Output flags.
return OK(out.str(), request.query.get("jsonp"));
@@ -311,6 +314,7 @@ Future<Response> state(
VLOG(1) << "HTTP request for '" << request.path << "'";
JSON::Object object;
+ object.values["version"] = MESOS_VERSION;
object.values["build_date"] = build::DATE;
object.values["build_time"] = build::TIME;
object.values["build_user"] = build::USER;
http://git-wip-us.apache.org/repos/asf/mesos/blob/e029cf50/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index 67877c9..e362c30 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -21,6 +21,8 @@
#include <string>
#include <vector>
+#include <mesos/mesos.hpp>
+
#include <stout/foreach.hpp>
#include <stout/json.hpp>
#include <stout/net.hpp>
@@ -261,9 +263,10 @@ Future<Response> vars(
std::ostringstream out;
out <<
- "build_date " << build::DATE << "\n" <<
- "build_user " << build::USER << "\n" <<
- "build_flags " << build::FLAGS << "\n";
+ "version: " << MESOS_VERSION << "\n" <<
+ "build_date: " << build::DATE << "\n" <<
+ "build_user: " << build::USER << "\n" <<
+ "build_flags: " << build::FLAGS << "\n";
// TODO(benh): Output flags.
return OK(out.str(), request.query.get("jsonp"));
@@ -301,6 +304,7 @@ Future<Response> state(
LOG(INFO) << "HTTP request for '" << request.path << "'";
JSON::Object object;
+ object.values["version"] = MESOS_VERSION;
object.values["build_date"] = build::DATE;
object.values["build_time"] = build::TIME;
object.values["build_user"] = build::USER;
[10/23] git commit: Updated CHANGELOG for 0.13.0 (rc2).
Posted by vi...@apache.org.
Updated CHANGELOG for 0.13.0 (rc2).
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/09f4c11b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/09f4c11b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/09f4c11b
Branch: refs/heads/vinod/0.13.0
Commit: 09f4c11b820124a42c18d52ce80ad0e1fc5abc82
Parents: cfb83a4
Author: Vinod Kone <vi...@twitter.com>
Authored: Tue Jun 11 12:07:13 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Jun 11 12:08:51 2013 -0700
----------------------------------------------------------------------
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/09f4c11b/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index b070e22..c262c74 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -58,6 +58,7 @@ Release Notes - Mesos - Version 0.13.0
* [MESOS-481] - Slave needs to only inform the master about non-terminal executors, for proper resource accounting
* [MESOS-482] - Status update manager should not cleanup the stream when there are pending updates, even though it received an ACK for a terminal update
* [MESOS-484] - Latest ZooKeeperState.cpp doesn't compile on Mountain Lion
+ * [MESOS-502] - Slave crashes when handling duplicate terminal updates
** Improvement
* [MESOS-46] - Refactor MasterTest to use fixture
[21/23] git commit: Fixed slave to properly handle terminated tasks
that have pending updates.
Posted by vi...@apache.org.
Fixed slave to properly handle terminated tasks that have pending
updates.
Review: https://reviews.apache.org/r/13187
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/93784622
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/93784622
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/93784622
Branch: refs/heads/vinod/0.13.0
Commit: 93784622e0e8557e09feab8dd446594c15b5b3fc
Parents: 5c19b04
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed Jul 31 22:26:49 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Aug 5 11:22:24 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 31 +++++++++++++-------------
src/slave/slave.hpp | 4 ++--
src/slave/status_update_manager.cpp | 12 +++++-----
src/slave/status_update_manager.hpp | 26 ++++++++++-----------
src/tests/slave_recovery_tests.cpp | 3 +--
src/tests/status_update_manager_tests.cpp | 6 -----
6 files changed, 38 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c56dafc..c40a9ac 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -850,7 +850,7 @@ void Slave::_runTask(
<< " because the framework is terminating";
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
return;
}
@@ -875,7 +875,7 @@ void Slave::_runTask(
statusUpdate(update);
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
return;
@@ -1152,7 +1152,7 @@ void Slave::shutdownFramework(const FrameworkID& frameworkId)
// NOTE: We call remove here to ensure we can remove an
// executor (of a terminating framework) that is terminated
// but waiting for acknowledgements.
- remove(framework, executor);
+ removeExecutor(framework, executor);
} else {
// Executor is terminating. Ignore.
}
@@ -1331,11 +1331,6 @@ void Slave::_statusUpdateAcknowledgement(
return;
}
- if (!future.get().get()) {
- // The status update manager ignored the acknowledgement (e.g., duplicate).
- return;
- }
-
VLOG(1) << "Status update manager successfully handled status update"
<< " acknowledgement " << uuid
<< " for task " << taskId
@@ -1372,14 +1367,16 @@ void Slave::_statusUpdateAcknowledgement(
executor->state == Executor::TERMINATED)
<< executor->state;
- if (executor->terminatedTasks.contains(taskId)) {
+ // If the task has reached terminal state and all its updates have
+ // been acknowledged, mark it completed.
+ if (executor->terminatedTasks.contains(taskId) && !future.get().get()) {
executor->completeTask(taskId);
}
// Remove the executor if it has terminated and there are no more
// incomplete tasks.
if (executor->state == Executor::TERMINATED && !executor->incompleteTasks()) {
- remove(framework, executor);
+ removeExecutor(framework, executor);
}
}
@@ -2161,7 +2158,7 @@ void Slave::executorTerminated(
// there are no incomplete tasks.
if (framework->state == Framework::TERMINATING ||
!executor->incompleteTasks()) {
- remove(framework, executor);
+ removeExecutor(framework, executor);
}
break;
}
@@ -2174,7 +2171,7 @@ void Slave::executorTerminated(
}
-void Slave::remove(Framework* framework, Executor* executor)
+void Slave::removeExecutor(Framework* framework, Executor* executor)
{
CHECK_NOTNULL(framework);
CHECK_NOTNULL(executor);
@@ -2234,12 +2231,12 @@ void Slave::remove(Framework* framework, Executor* executor)
// Remove this framework if it has no pending executors and tasks.
if (framework->executors.empty() && framework->pending.empty()) {
- remove(framework);
+ removeFramework(framework);
}
}
-void Slave::remove(Framework* framework)
+void Slave::removeFramework(Framework* framework)
{
CHECK_NOTNULL(framework);
@@ -2665,7 +2662,7 @@ void Slave::recoverFramework(const FrameworkState& state, bool reconnect)
// Remove the framework in case we didn't recover any executors.
if (framework->executors.empty()) {
- remove(framework);
+ removeFramework(framework);
}
}
@@ -2950,6 +2947,8 @@ void Executor::terminateTask(
const TaskID& taskId,
const mesos::TaskState& state)
{
+ VLOG(1) << "Terminating task " << taskId;
+
Task* task = NULL;
// Remove the task if it's queued.
if (queuedTasks.contains(taskId)) {
@@ -2970,6 +2969,8 @@ void Executor::terminateTask(
void Executor::completeTask(const TaskID& taskId)
{
+ VLOG(1) << "Completing task " << taskId;
+
CHECK(terminatedTasks.contains(taskId))
<< "Failed to find terminated task " << taskId;
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index a9fd9c9..010159c 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -246,10 +246,10 @@ protected:
void recoverFramework(const state::FrameworkState& state, bool reconnect);
// Removes and garbage collects the executor.
- void remove(Framework* framework, Executor* executor);
+ void removeExecutor(Framework* framework, Executor* executor);
// Removes and garbage collects the framework.
- void remove(Framework* framework);
+ void removeFramework(Framework* framework);
private:
Slave(const Slave&); // No copying.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.cpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.cpp b/src/slave/status_update_manager.cpp
index 9e9e4e2..fbc09e6 100644
--- a/src/slave/status_update_manager.cpp
+++ b/src/slave/status_update_manager.cpp
@@ -380,17 +380,17 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
// This might happen if we retried a status update and got back
// acknowledgments for both the original and the retried update.
if (update.isNone()) {
- LOG(WARNING) << "Ignoring unexpected status update acknowledgment " << uuid
- << " for task " << taskId
- << " of framework " << frameworkId;
- return false;
+ return Error(
+ "Unexpected status update acknowledgment " + uuid.toString() +
+ " for task " + stringify(taskId) +
+ " of framework " + stringify(frameworkId));
}
// Handle the acknowledgement.
Try<bool> result =
stream->acknowledgement(taskId, frameworkId, uuid, update.get());
- if (result.isError() || !result.get()) {
+ if (result.isError()) {
return result;
}
@@ -415,7 +415,7 @@ Try<bool> StatusUpdateManagerProcess::acknowledgement(
stream->timeout = forward(next.get());
}
- return true;
+ return result;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/slave/status_update_manager.hpp
----------------------------------------------------------------------
diff --git a/src/slave/status_update_manager.hpp b/src/slave/status_update_manager.hpp
index 795e74c..d166e29 100644
--- a/src/slave/status_update_manager.hpp
+++ b/src/slave/status_update_manager.hpp
@@ -36,6 +36,7 @@
#include <stout/option.hpp>
#include <stout/os.hpp>
#include <stout/protobuf.hpp>
+#include <stout/stringify.hpp>
#include <stout/utils.hpp>
#include <stout/uuid.hpp>
@@ -102,9 +103,10 @@ public:
// Checkpoints the status update to disk if necessary.
// Also, sends the next pending status update, if any.
- // @return True if the ACK is handled successfully (e.g., checkpointed).
- // False if the ACK was ignored because it was a duplicate.
- // Error if there are any errors (e.g., checkpointing).
+ // @return true if the ACK is handled successfully (e.g., checkpointed)
+ // and the task's status update stream is not terminated.
+ // false same as above except the status update stream is terminated.
+ // Error if there are any errors (e.g., duplicate, checkpointing).
process::Future<Try<bool> > acknowledgement(
const TaskID& taskId,
const FrameworkID& frameworkId,
@@ -236,20 +238,18 @@ struct StatusUpdateStream
}
if (acknowledged.contains(uuid)) {
- LOG(WARNING) << "Ignoring duplicate status update acknowledgment " << uuid
- << " for update " << update;
- return false;
+ return Error("Duplicate status update acknowledgment " + uuid.toString() +
+ " for update " + stringify(update));
}
// This might happen if we retried a status update and got back
// acknowledgments for both the original and the retried update.
if (uuid != UUID::fromBytes(update.uuid())) {
- LOG(WARNING)
- << "Ignoring unexpected status update acknowledgement "
- << "(received " << uuid
- << ", expecting " << UUID::fromBytes(update.uuid())
- << ") for update " << update;
- return false;
+ return Error(
+ "Unexpected status update acknowledgement (received " +
+ uuid.toString() +
+ ", expecting " + UUID::fromBytes(update.uuid()).toString() +
+ ") for update " + stringify(update));
}
// Handle the ACK, checkpointing if necessary.
@@ -258,7 +258,7 @@ struct StatusUpdateStream
return Error(result.error());
}
- return true;
+ return !terminated;
}
// Returns the next update (or none, if empty) in the queue.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index daae66e..02425f6 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -902,8 +902,7 @@ TYPED_TEST(SlaveRecoveryTest, NonCheckpointingFramework)
Clock::pause();
- Future<Nothing> updateFramework =
- FUTURE_DISPATCH(_, &Slave::updateFramework);
+ Future<Nothing> updateFramework = FUTURE_DISPATCH(_, &Slave::updateFramework);
// Simulate a 'UpdateFrameworkMessage' to ensure framework pid is
// not being checkpointed.
http://git-wip-us.apache.org/repos/asf/mesos/blob/93784622/src/tests/status_update_manager_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/status_update_manager_tests.cpp b/src/tests/status_update_manager_tests.cpp
index 4239532..6473478 100644
--- a/src/tests/status_update_manager_tests.cpp
+++ b/src/tests/status_update_manager_tests.cpp
@@ -357,9 +357,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreDuplicateStatusUpdateAck)
update.status().task_id(),
update.uuid());
- // TODO(vinod): It would've been great to introspect the first
- // argument of '_statusUpdateAcknowledement()' and ensure that
- // it is 'false'.
AWAIT_READY(duplicateAck);
Clock::resume();
@@ -451,9 +448,6 @@ TEST_F(StatusUpdateManagerTest, IgnoreUnexpectedStatusUpdateAck)
update.status().task_id(),
UUID::random().toBytes());
- // TODO(vinod): It would've been great to introspect the first
- // argument of '_statusUpdateAcknowledement()' and ensure that
- // it is 'false'.
AWAIT_READY(unexpectedAck);
EXPECT_CALL(exec, shutdown(_))
[08/23] git commit: Updated CHANGELOG for 0.13.0.
Posted by vi...@apache.org.
Updated CHANGELOG for 0.13.0.
Conflicts:
CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bc2a09d1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bc2a09d1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bc2a09d1
Branch: refs/heads/vinod/0.13.0
Commit: bc2a09d11edd34a66d438badb17700e5728b40df
Parents: 062c850
Author: Vinod Kone <vi...@twitter.com>
Authored: Mon Jun 10 14:47:35 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Jun 11 12:01:23 2013 -0700
----------------------------------------------------------------------
CHANGELOG | 159 ++++++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 130 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/bc2a09d1/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 844696e..b070e22 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,115 @@
+Release Notes - Mesos - Version 0.13.0
+--------------------------------------
+* The primary feature in this release is "Slave Recovery". This enables a slave to reconnect
+ with old running executors after it restarts (e.g., after an upgrade).
+* This release also includes a major refactor of the internal testing infrastructure.
+* There are also several bug fixes and stability improvements (esp. around ZooKeeper).
+
+** Bug
+ * [MESOS-77] - ExceptionTest.AbortOnFrameworkError sometimes hangs if Mesos built without optimizations
+ * [MESOS-201] - CppFramework test occasionally fails
+ * [MESOS-217] - LOST tasks are incorrectly reconciled between mesos and framework
+ * [MESOS-232] - Unit test CoordinatorTest.Elect triggers non-deterministic assertion failure in libprocess.
+ * [MESOS-276] - SIGSEV with current trunk and OpenJDK 7u3
+ * [MESOS-277] - Java test framework test is flaky
+ * [MESOS-289] - Zookeeper tests are flaky
+ * [MESOS-301] - Coordinator test is flaky
+ * [MESOS-318] - os::memory does not consider sysinfo.mem_unit
+ * [MESOS-321] - libprocess http::encode fails test
+ * [MESOS-344] - --disable-java still uses java headers during make check
+ * [MESOS-353] - ZooKeepet state test GetSetGet hung
+ * [MESOS-362] - Inconsistent slave maps in the master.
+ * [MESOS-365] - Slave should reject tasks before registering with the master.
+ * [MESOS-366] - Master check failure during load tests.
+ * [MESOS-368] - HTTP.Endpoints test is flaky.
+ * [MESOS-369] - Mesos tests spitting out error messages.
+ * [MESOS-379] - Zookeeper MasterDetectorExpireSlaveZKSessionNewMaster test is flaky
+ * [MESOS-385] - MasterTest.TaskRunning flaky on Jenkins.
+ * [MESOS-392] - FaultTolerance SchedulerExit test hangs
+ * [MESOS-393] - Forking at an unlucky time on OS X can cause the C++ library to deadlock.
+ * [MESOS-394] - Don't do ExecutorLauncher in forked process but exec first instead.
+ * [MESOS-395] - FaultToleranceTest.SchedulerFailoverFrameworkMessage test is flaky.
+ * [MESOS-399] - MonitorTest.WatchUnwatch failed.
+ * [MESOS-400] - Example Java framework test is flaky
+ * [MESOS-401] - SlaveRecoveryTest/0.RecoverTerminatedExecutor is flaky on OSX.
+ * [MESOS-402] - CoordinatorTest.TruncateNotLearnedFill test is flaky
+ * [MESOS-403] - CoordinatorTest.TruncateLearnedFill test is flaky
+ * [MESOS-405] - SlaveRecoveryTest/1.ReconnectExecutor crashes.
+ * [MESOS-406] - Google mock throws a segfault when invoked by TestFilter
+ * [MESOS-407] - Google test filter processing is incorrect for the empty string.
+ * [MESOS-408] - FaultToleranceTest.SlavePartitioned is flaky
+ * [MESOS-412] - MasterTest.ShutdownUnregisteredExecutor flaky
+ * [MESOS-423] - A slave asked to shutdown should not re-register with a new slave id
+ * [MESOS-424] - CgroupsIsolatorTest.BalloonFramework runs forever
+ * [MESOS-436] - FaultToleranceTest.SchedulerFailover test is flaky
+ * [MESOS-437] - ResourceOffersTest.ResourceOfferWithMultipleSlaves is flaky
+ * [MESOS-440] - Allow for headroom in the GC algorithm.
+ * [MESOS-441] - AllocatorZooKeeperTest/0.FrameworkReregistersFirst is flaky
+ * [MESOS-446] - Master should shutdown slaves that were deactivated
+ * [MESOS-447] - Master should send TASK_LOST updates for unknown tasks when slave reregisters
+ * [MESOS-453] - AllocatorZookeeper tests are using /tmp/mesos work directory
+ * [MESOS-454] - ResourceOffers tests are using /tmp/mesos working directory
+ * [MESOS-462] - Resource usage collection failure messages have '1' as the failure message.
+ * [MESOS-466] - Master should always send a re-registered message to framework when the scheduler driver sends a re-register
+ * [MESOS-467] - AllocatorTest.FrameworkExited is flaky
+ * [MESOS-469] - Scheduler driver should call disconnected on master failover
+ * [MESOS-474] - Mesos 0.10.0: make check fails on Ubuntu 12.04LTS
+ * [MESOS-476] - Upgrade libev to 4.15
+ * [MESOS-481] - Slave needs to only inform the master about non-terminal executors, for proper resource accounting
+ * [MESOS-482] - Status update manager should not cleanup the stream when there are pending updates, even though it received an ACK for a terminal update
+ * [MESOS-484] - Latest ZooKeeperState.cpp doesn't compile on Mountain Lion
+
+** Improvement
+ * [MESOS-46] - Refactor MasterTest to use fixture
+ * [MESOS-134] - Add Python documentation
+ * [MESOS-140] - Unrecognized command line args should fail the process
+ * [MESOS-242] - Add more tests to Dominant Share Allocator
+ * [MESOS-305] - Inform the frameworks / slaves about a master failover
+ * [MESOS-409] - Master detector code should stat nodes before attempting to create
+ * [MESOS-472] - Separate ResourceStatistics::cpu_time into ResourceStatistics::cpu_user_time and ResourceStatistics::cpu_system_time.
+ * [MESOS-493] - Expose version information in http endpoints
+
+** New Feature
+ * [MESOS-110] - Mesos deploys should not restart tasks
+ * [MESOS-169] - Ability to for tests to catch in-flight messages that are dispatched
+
+
+Release Notes - Mesos - Version 0.12.0
+--------------------------------------
+* This release includes bug fixes and stability improvements.
+* The primary feature in this release is executor resource consumption monitoring. Slaves now monitor resource consumption of running executors and expose it over JSON and through the webui.
+* This release also includes a new and improved Hadoop framework. The new port doesn't require patching Hadoop (i.e., it's a self contained Hadoop contrib) and lets you use existing schedulers (e.g., the fair scheduler or capacity scheduler)! The tradeoff, however, is that it doesn't take as much advantage of the fine-grained nature of a Mesos task (i.e., there is no longer a 1-1 mapping between a Mesos task and a map/reduce task).
+
+** Sub-task
+ * [MESOS-214] - Report resources being used by executors
+ * [MESOS-419] - Old slave directories should not be garbage collected based on file modification time
+
+** Bug
+ * [MESOS-107] - Scheduler library should not acknowledge a status update if the driver has been aborted.
+ * [MESOS-152] - Slave should forward status updates for unknown tasks
+ * [MESOS-285] - configure.macosx checks for version "10.7" but should check for 10.7 or greater
+ * [MESOS-307] - Web UI file download links are broken.
+ * [MESOS-317] - python mesos core bindings rejects framework messages with null bytes
+ * [MESOS-319] - Fix buggy read / write calls.
+ * [MESOS-325] - make clean is broken
+ * [MESOS-332] - Executor launcher fetches resources as slave user, instead of executor user.
+ * [MESOS-340] - Gperftools target always rebuilds.
+ * [MESOS-374] - HTTP GET requests to /statistics/snapshot.json crash the slave
+ * [MESOS-422] - Master leader election should be more robust to stale ephemeral nodes
+ * [MESOS-486] - TaskInfo should include a 'source' in order to enable getting resource monitoring statistics.
+
+** Improvement
+ * [MESOS-293] - Make clean deletes checked in files.
+
+** New Feature
+ * [MESOS-99] - display slave resource usage information in the slave webui
+
+** Task
+ * [MESOS-274] - Unicode / Binary files over http endpoints.
+ * [MESOS-324] - Monitor executor resource usage.
+ * [MESOS-331] - Add --disable-perftools to configure.
+
+
Release Notes - Mesos - Version 0.11.0
--------------------------------------
** Brainstorming
@@ -6,14 +118,27 @@ Release Notes - Mesos - Version 0.11.0
** Bug
* [MESOS-260] - Implement a duration abstraction
+ * [MESOS-261] - bootstrap fails when automake version >= 1.12
+ * [MESOS-263] - Complete the new webui (slave, framework, executor pages)
+ * [MESOS-264] - Make fails on the latest ubuntu
+ * [MESOS-270] - Log viewing broken on mesos-local runs
* [MESOS-364] - cgroup tests fail on Ubuntu
* [MESOS-386] - AllocatorTest/0.TaskFinished has incomplete expectations.
* [MESOS-388] - Latest update breaks building on OSX
* [MESOS-404] - FilesTest.BrowseTest is flaky
* [MESOS-413] - AllocatorTest/0.TaskFinished test has bad expectations.
+** Improvement
+ * [MESOS-252] - Web UI Improvements
+ * [MESOS-253] - Enable -Wall -Werror on the build
+ * [MESOS-254] - Improve mesos slave's garbage collection
+ * [MESOS-259] - Expose slave attributes in slave endpoint & sortable tables
+
** Task
+ * [MESOS-275] - HTTP endpoint for file download.
+ * [MESOS-279] - Impose a limit on HTTP Response size
* [MESOS-389] - Add OSX slave to the Jenkins build.
+ * [MESOS-491] - Add mesos-0.11.0-incubating jar to maven central
Release Notes - Mesos - Version 0.10.0
@@ -34,7 +159,6 @@ Release Notes - Mesos - Version 0.10.0
* [MESOS-83] - Filters should be removed when more resources are available than rejected offer had
* [MESOS-145] - mesos executor holds on to fd spawned by slave after slave death, preventing slave from restarting
* [MESOS-148] - Building of included Hadoop broken
- * [MESOS-152] - Slave should forward status updates for unknown tasks
* [MESOS-164] - Crash on Mac OS X due to dlopen not being thread-safe
* [MESOS-183] - Included MPI Framework Fails to Start
* [MESOS-187] - Mesos should not pass an invalid task to a slave
@@ -53,16 +177,11 @@ Release Notes - Mesos - Version 0.10.0
* [MESOS-248] - Python quit unexpectedly while using mesos plugin
* [MESOS-251] - DRF allocator doesn't expire filter correctly
* [MESOS-257] - Master doesn't recover resources when executor exits
- * [MESOS-261] - bootstrap fails when automake version >= 1.12
- * [MESOS-262] - Slave should not charge the resources required for launching a executor against the executor
- * [MESOS-263] - Complete the new webui (slave, framework, executor pages)
- * [MESOS-264] - Make fails on the latest ubuntu
+ * [MESOS-262] - Slave should not charge the resources required for launching a executor against the executor
* [MESOS-266] - When the master removes a slave, a shutdown should be sent.
- * [MESOS-268] - Slave should force kill executors when it is shutting down
- * [MESOS-270] - Log viewing broken on mesos-local runs
+ * [MESOS-268] - Slave should force kill executors when it is shutting down
* [MESOS-278] - Master Fails to Connect to Zookeeper
* [MESOS-284] - Short-term fix for fire-walling slave shutdown and lost slave messages from the 'wrong' master
- * [MESOS-285] - configure.macosx checks for version "10.7" but should check for 10.7 or greater
* [MESOS-286] - AllocatorTest is flaky
* [MESOS-288] - Latest trunk does not finish make check
* [MESOS-290] - Jobtracker can't get TaskTrackerInfo when the JobTracker log file is deleted
@@ -70,18 +189,11 @@ Release Notes - Mesos - Version 0.10.0
* [MESOS-302] - Scheduler driver shouldn't send an ACK if the driver is aborted while sending stats update
* [MESOS-303] - mesos slave crashes during framework termination
* [MESOS-306] - Mesos-master frequently crashes
- * [MESOS-307] - Web UI file download links are broken.
* [MESOS-310] - cgroups isolation module should not block on fetching executors
- * [MESOS-317] - python mesos core bindings rejects framework messages with null bytes
- * [MESOS-319] - Fix buggy read / write calls.
- * [MESOS-321] - libprocess http::encode fails test
- * [MESOS-325] - make clean is broken
- * [MESOS-332] - Executor launcher fetches resources as slave user, instead of executor user.
* [MESOS-339] - Release script is expecting enter, not any key
- * [MESOS-340] - Gperftools target always rebuilds.
- * [MESOS-381] - Problem building on Ubuntu 12.04 LTE
* [MESOS-382] - FaultToleranceTest.FrameworkReliableRegistration test is flaky.
* [MESOS-383] - AllocatorTest/0.FrameworkExited test is broken
+ * [MESOS-464] - mesos 0.10.0 fails to build on ubutu 13.04
** Improvement
* [MESOS-8] - Maintain a history of executed frameworks/tasks and show it on the web UI
@@ -90,20 +202,14 @@ Release Notes - Mesos - Version 0.10.0
* [MESOS-149] - Garbage collection on slaves
* [MESOS-171] - Make CommandInfo 'uri' field be repeated, possibly making a URI embedded message to describe whether or not we should 'chmod +x' the resulting resource.
* [MESOS-180] - Update the Hadoop patch to list protobuf-2.4.1 as a dependency so Maven pulls it down.
- * [MESOS-193] - Create a single-page javascript interface to replace the existing webui
+ * [MESOS-193] - Create a single-page javascript interface to replace the existing webui
* [MESOS-194] - Make killtree more verbose.
- * [MESOS-252] - Web UI Improvements
- * [MESOS-253] - Enable -Wall -Werror on the build
- * [MESOS-254] - Improve mesos slave's garbage collection
* [MESOS-255] - Expose files through HTTP endpoints.
* [MESOS-256] - Introduce a cluster name into the Web UI
- * [MESOS-259] - Expose slave attributes in slave endpoint & sortable tables
* [MESOS-272] - Create a 'fs' namespace and migrate as appropriate from our 'os' namespace.
- * [MESOS-293] - Make clean deletes checked in files.
** New Feature
* [MESOS-86] - Expose master url to the scheduler
- * [MESOS-99] - display slave resource usage information in the slave webui
* [MESOS-158] - Make ExecutorInfo more rich
* [MESOS-185] - Provide a master stat indicating number of outstanding resource offers
* [MESOS-207] - A new isolation module on Linux that uses Linux control groups (cgroups) directly.
@@ -118,12 +224,7 @@ Release Notes - Mesos - Version 0.10.0
* [MESOS-69] - Migrate to Apache wiki and de-activate github wiki
* [MESOS-80] - Add a page or section to the wiki defining project coding standards
* [MESOS-133] - Make Mesos clean of most GCC warnings
- * [MESOS-274] - Unicode / Binary files over http endpoints.
- * [MESOS-275] - HTTP endpoint for file download.
- * [MESOS-279] - Impose a limit on HTTP Response size
- * [MESOS-283] - Use cpp implementations for python protobufs.
- * [MESOS-324] - Monitor executor resource usage.
- * [MESOS-331] - Add --disable-perftools to configure.
+ * [MESOS-398] - Add mesos-0.10.0-incubating jar to maven central
Release Notes - Mesos - Version 0.9.0
[09/23] git commit: Fixed slave to properly handle duplicate terminal
updates for the same task.
Posted by vi...@apache.org.
Fixed slave to properly handle duplicate terminal updates for the
same task.
Review: https://reviews.apache.org/r/11798
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cfb83a4f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cfb83a4f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cfb83a4f
Branch: refs/heads/vinod/0.13.0
Commit: cfb83a4f98c67c3fd74ab4375f474989763d0b2e
Parents: bc2a09d
Author: Vinod Kone <vi...@twitter.com>
Authored: Mon Jun 10 22:44:42 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Jun 11 12:01:44 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 25 ++-
src/tests/status_update_manager_tests.cpp | 202 +++++++++++++++++++++++++
2 files changed, 220 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cfb83a4f/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index b5b7e0e..fd96de9 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -1687,6 +1687,8 @@ void Slave::reregisterExecutorTimeout()
// 2) When slave generates task updates (e.g LOST/KILLED/FAILED).
void Slave::statusUpdate(const StatusUpdate& update)
{
+ LOG(INFO) << "Handling status update " << update;
+
CHECK(state == RECOVERING || state == DISCONNECTED ||
state == RUNNING || state == TERMINATING)
<< state;
@@ -1724,7 +1726,8 @@ void Slave::statusUpdate(const StatusUpdate& update)
// generated by the slave when the executor is unknown to it,
// e.g., killTask(), _runTask().
// TODO(vinod): Revisit these semantics when we disallow updates
- // sent by executors that are unknown to the slave.
+ // (e.g., when slave recovery is always enabled) sent by executors
+ // that are unknown to the slave.
statusUpdateManager->update(update, info.id())
.onAny(defer(self(), &Slave::_statusUpdate, params::_1, update, None()));
@@ -1737,15 +1740,17 @@ void Slave::statusUpdate(const StatusUpdate& update)
executor->state == Executor::TERMINATED)
<< executor->state;
- VLOG(1) << "Handling status update " << update;
-
stats.tasks[update.status().state()]++;
stats.validStatusUpdates++;
executor->updateTaskState(status.task_id(), status.state());
- // Handle the task appropriately if it's terminated.
- if (protobuf::isTerminalState(status.state())) {
+ // Handle the task appropriately if it is terminated.
+ // TODO(vinod): Revisit these semantics when we disallow duplicate
+ // terminal updates (e.g., when slave recovery is always enabled).
+ if (protobuf::isTerminalState(status.state()) &&
+ (executor->queuedTasks.contains(status.task_id()) ||
+ executor->launchedTasks.contains(status.task_id()))) {
executor->terminateTask(status.task_id(), status.state());
// Tell the isolator to update the resources.
@@ -2967,7 +2972,8 @@ void Executor::terminateTask(
void Executor::completeTask(const TaskID& taskId)
{
- CHECK(terminatedTasks.contains(taskId)) << "Unknown task " << taskId;
+ CHECK(terminatedTasks.contains(taskId))
+ << "Failed to find terminated task " << taskId;
Task* task = terminatedTasks[taskId];
completedTasks.push_back(*task);
@@ -3018,7 +3024,12 @@ void Executor::recoverTask(const TaskState& state)
updateTaskState(state.id, update.status().state());
// Terminate the task if it received a terminal update.
- if (protobuf::isTerminalState(update.status().state())) {
+ // We ignore duplicate terminal updates by checking if
+ // the task is present in launchedTasks.
+ // TODO(vinod): Revisit these semantics when we disallow duplicate
+ // terminal updates (e.g., when slave recovery is always enabled).
+ if (protobuf::isTerminalState(update.status().state()) &&
+ launchedTasks.contains(state.id)) {
terminateTask(state.id, update.status().state());
// If the terminal update has been acknowledged, remove it.
http://git-wip-us.apache.org/repos/asf/mesos/blob/cfb83a4f/src/tests/status_update_manager_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/status_update_manager_tests.cpp b/src/tests/status_update_manager_tests.cpp
index 61ccfcc..4239532 100644
--- a/src/tests/status_update_manager_tests.cpp
+++ b/src/tests/status_update_manager_tests.cpp
@@ -464,3 +464,205 @@ TEST_F(StatusUpdateManagerTest, IgnoreUnexpectedStatusUpdateAck)
Shutdown();
}
+
+
+// This test verifies that the slave and status update manager
+// properly handle duplicate terminal status updates, when the
+// second update is received before the ACK for the first update.
+// The proper behavior here is for the status update manager to
+// drop the duplicate update.
+TEST_F(StatusUpdateManagerTest, DuplicateTerminalUpdateBeforeAck)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.checkpoint = true;
+
+ Try<PID<Slave> > slave = StartSlave(&exec, flags);
+ ASSERT_SOME(slave);
+
+ FrameworkInfo frameworkInfo; // Bug in gcc 4.1.*, must assign on next line.
+ frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_checkpoint(true); // Enable checkpointing.
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(&sched, frameworkInfo, master.get());
+
+ FrameworkID frameworkId;
+ EXPECT_CALL(sched, registered(_, _, _))
+ .WillOnce(SaveArg<1>(&frameworkId));
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(_, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ ExecutorDriver* execDriver;
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .WillOnce(SaveArg<0>(&execDriver));
+
+ // Send a terminal update right away.
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(_, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ // Drop the first ACK from the scheduler to the slave.
+ Future<StatusUpdateAcknowledgementMessage> statusUpdateAcknowledgementMessage =
+ DROP_PROTOBUF(StatusUpdateAcknowledgementMessage(), _, slave.get());
+
+ Clock::pause();
+
+ driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0]));
+
+ AWAIT_READY(status);
+
+ EXPECT_EQ(TASK_FINISHED, status.get().state());
+
+ AWAIT_READY(statusUpdateAcknowledgementMessage);
+
+ Future<Nothing> _statusUpdate =
+ FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdate);
+
+ // Now send a TASK_KILLED update for the same task.
+ TaskStatus status2 = status.get();
+ status2.set_state(TASK_KILLED);
+ execDriver->sendStatusUpdate(status2);
+
+ // At this point the status update manager has enqueued
+ // TASK_FINISHED and TASK_KILLED updates.
+ AWAIT_READY(_statusUpdate);
+
+ // After we advance the clock, the scheduler should receive
+ // the retried TASK_FINISHED update and acknowledge it. The
+ // TASK_KILLED update should be dropped by the status update
+ // manager, as the stream is already terminated.
+ Future<TaskStatus> update;
+ EXPECT_CALL(sched, statusUpdate(_, _))
+ .WillOnce(FutureArg<1>(&update));
+
+ Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL);
+ Clock::settle();
+
+ // Ensure the scheduler receives TASK_FINISHED.
+ AWAIT_READY(update);
+ EXPECT_EQ(TASK_FINISHED, update.get().state());
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ Clock::resume();
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
+
+
+// This test verifies that the slave and status update manager
+// properly handle duplicate terminal status updates, when the
+// second update is received after the ACK for the first update.
+// The proper behavior here is for the status update manager to
+// forward the duplicate update to the scheduler.
+TEST_F(StatusUpdateManagerTest, DuplicateTerminalUpdateAfterAck)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.checkpoint = true;
+
+ Try<PID<Slave> > slave = StartSlave(&exec, flags);
+ ASSERT_SOME(slave);
+
+ FrameworkInfo frameworkInfo; // Bug in gcc 4.1.*, must assign on next line.
+ frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_checkpoint(true); // Enable checkpointing.
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(&sched, frameworkInfo, master.get());
+
+ FrameworkID frameworkId;
+ EXPECT_CALL(sched, registered(_, _, _))
+ .WillOnce(SaveArg<1>(&frameworkId));
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(_, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ ExecutorDriver* execDriver;
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .WillOnce(SaveArg<0>(&execDriver));
+
+ // Send a terminal update right away.
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(_, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ Future<Nothing> _statusUpdateAcknowledgement =
+ FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement);
+
+ driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0]));
+
+ AWAIT_READY(status);
+
+ EXPECT_EQ(TASK_FINISHED, status.get().state());
+
+ AWAIT_READY(_statusUpdateAcknowledgement);
+
+ Future<TaskStatus> update;
+ EXPECT_CALL(sched, statusUpdate(_, _))
+ .WillOnce(FutureArg<1>(&update));
+
+ Future<Nothing> _statusUpdateAcknowledgement2 =
+ FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement);
+
+ Clock::pause();
+
+ // Now send a TASK_KILLED update for the same task.
+ TaskStatus status2 = status.get();
+ status2.set_state(TASK_KILLED);
+ execDriver->sendStatusUpdate(status2);
+
+ // Ensure the scheduler receives TASK_KILLED.
+ AWAIT_READY(update);
+ EXPECT_EQ(TASK_KILLED, update.get().state());
+
+ // Ensure the slave properly handles the ACK.
+ // Clock::settle() ensures that the slave successfully
+ // executes Slave::_statusUpdateAcknowledgement().
+ AWAIT_READY(_statusUpdateAcknowledgement2);
+ Clock::settle();
+
+ Clock::resume();
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
[03/23] git commit: Updated the NOTICE to include the correct year,
and to fix line wrapping.
Posted by vi...@apache.org.
Updated the NOTICE to include the correct year, and to fix line
wrapping.
Review: https://reviews.apache.org/r/11696
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3f809e07
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3f809e07
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3f809e07
Branch: refs/heads/vinod/0.13.0
Commit: 3f809e076467c785d3877ede9e960ecdf789d9a5
Parents: 5912aa2
Author: Benjamin Mahler <bm...@twitter.com>
Authored: Thu Jun 6 15:17:57 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
NOTICE | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3f809e07/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index f92cbb4..aa2fc8b 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Mesos
-Copyright 2012, The Apache Software Foundation
+Copyright 2013, The Apache Software Foundation
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
[22/23] git commit: Fixed slave to remove a queued task when the task
is terminated.
Posted by vi...@apache.org.
Fixed slave to remove a queued task when the task is terminated.
Review: https://reviews.apache.org/r/13207
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/88b589cc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/88b589cc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/88b589cc
Branch: refs/heads/vinod/0.13.0
Commit: 88b589cc8a3cbacfc764e1629a4bff0622238d4b
Parents: 9378462
Author: Vinod Kone <vi...@twitter.com>
Authored: Thu Aug 1 17:03:04 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Aug 5 11:26:33 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 1 +
src/tests/master_tests.cpp | 79 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 80 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b589cc/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index c40a9ac..20d76e2 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -2954,6 +2954,7 @@ void Executor::terminateTask(
if (queuedTasks.contains(taskId)) {
task = new Task(
protobuf::createTask(queuedTasks[taskId], state, id, frameworkId));
+ queuedTasks.erase(taskId);
} else if (launchedTasks.contains(taskId)) {
// Update the resources if it's been launched.
task = launchedTasks[taskId];
http://git-wip-us.apache.org/repos/asf/mesos/blob/88b589cc/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 5ac4d5f..c503842 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -40,6 +40,7 @@
#include "master/master.hpp"
#include "slave/constants.hpp"
+#include "slave/gc.hpp"
#include "slave/flags.hpp"
#include "slave/process_isolator.hpp"
#include "slave/slave.hpp"
@@ -53,6 +54,7 @@ using namespace mesos::internal::tests;
using mesos::internal::master::Master;
+using mesos::internal::slave::GarbageCollectorProcess;
using mesos::internal::slave::Isolator;
using mesos::internal::slave::ProcessIsolator;
using mesos::internal::slave::Slave;
@@ -755,6 +757,83 @@ TEST_F(MasterTest, ShutdownUnregisteredExecutor)
}
+// This test verifies that when an executor terminates before
+// registering with slave, it is properly cleaned up.
+TEST_F(MasterTest, RemoveUnregisteredTerminatedExecutor)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+ TestingIsolator isolator(&exec);
+
+ Try<PID<Slave> > slave = StartSlave(&isolator);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(&sched, DEFAULT_FRAMEWORK_INFO, master.get());
+
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(1);
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
+ task.mutable_resources()->MergeFrom(offers.get()[0].resources());
+ task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ // Drop the registration message from the executor to the slave.
+ Future<process::Message> registerExecutorMessage =
+ DROP_MESSAGE(Eq(RegisterExecutorMessage().GetTypeName()), _, _);
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY(registerExecutorMessage);
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ Future<Nothing> schedule =
+ FUTURE_DISPATCH(_, &GarbageCollectorProcess::schedule);
+
+ // Now kill the executor.
+ dispatch(isolator,
+ &Isolator::killExecutor,
+ offers.get()[0].framework_id(),
+ DEFAULT_EXECUTOR_ID);
+
+ AWAIT_READY(status);
+ EXPECT_EQ(TASK_LOST, status.get().state());
+
+ // We use 'gc.schedule' as a signal for the executor being cleaned
+ // up by the slave.
+ AWAIT_READY(schedule);
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown(); // Must shutdown before 'isolator' gets deallocated.
+}
+
+
TEST_F(MasterTest, MasterInfo)
{
Try<PID<Master> > master = StartMaster();
[16/23] git commit: Fixed a bug in master to properly remove a non
checkpointing framework from a checkpointing slave.
Posted by vi...@apache.org.
Fixed a bug in master to properly remove a non checkpointing
framework from a checkpointing slave.
Review: https://reviews.apache.org/r/12182
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bf703d2a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bf703d2a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bf703d2a
Branch: refs/heads/vinod/0.13.0
Commit: bf703d2a1a0f30fb290c9a4809a7a919c8258504
Parents: a278aca
Author: Vinod Kone <vi...@twitter.com>
Authored: Fri Jun 28 14:40:49 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:47:20 2013 -0700
----------------------------------------------------------------------
src/master/master.cpp | 56 ++++++++++++++++++---------------
src/tests/slave_recovery_tests.cpp | 42 ++++++++++++++++++-------
2 files changed, 62 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf703d2a/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 5782164..e9f2225 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -468,19 +468,25 @@ void Master::exited(const UPID& pid)
} else {
// If a slave is checkpointing, remove all non-checkpointing
// frameworks from the slave.
- hashset<FrameworkID> ids;
- foreachvalue (Task* task, utils::copy(slave->tasks)) {
- if (!ids.contains(task->framework_id())) {
- ids.insert(task->framework_id());
- Framework* framework = getFramework(task->framework_id());
- if (framework != NULL && !framework->info.checkpoint()) {
- LOG(INFO) << "Removing non-checkpointing framework "
- << task->framework_id()
- << " from disconnected slave " << slave->id
- << "(" << slave->info.hostname() << ")";
-
- removeFramework(slave, framework);
- }
+
+ // First, collect all the frameworks running on this slave.
+ hashset<FrameworkID> frameworkIds;
+ foreachvalue (Task* task, slave->tasks) {
+ frameworkIds.insert(task->framework_id());
+ }
+ foreachkey (const FrameworkID& frameworkId, slave->executors) {
+ frameworkIds.insert(frameworkId);
+ }
+
+ // Now, remove all the non-checkpointing frameworks.
+ foreach (const FrameworkID& frameworkId, frameworkIds) {
+ Framework* framework = getFramework(frameworkId);
+ if (framework != NULL && !framework->info.checkpoint()) {
+ LOG(INFO) << "Removing non-checkpointing framework " << frameworkId
+ << " from disconnected slave " << slave->id
+ << "(" << slave->info.hostname() << ")";
+
+ removeFramework(slave, framework);
}
}
}
@@ -1790,20 +1796,20 @@ void Master::removeFramework(Slave* slave, Framework* framework)
// over and the framework hasn't reconnected yet. For more info
// please see the comments in 'removeFramework(Framework*)'.
StatusUpdateMessage message;
- StatusUpdate* update = message.mutable_update();
- update->mutable_framework_id()->MergeFrom(task->framework_id());
+ message.mutable_update()->CopyFrom(
+ protobuf::createStatusUpdate(
+ task->framework_id(),
+ task->slave_id(),
+ task->task_id(),
+ TASK_LOST,
+ "Slave " + slave->info.hostname() + " disconnected",
+ (task->has_executor_id() ?
+ Option<ExecutorID>(task->executor_id()) : None())));
- if (task->has_executor_id()) {
- update->mutable_executor_id()->MergeFrom(task->executor_id());
- }
+ LOG(INFO) << "Sending status update " << message.update()
+ << " due to disconnected slave " << slave->id
+ << " (" << slave->info.hostname() << ")";
- update->mutable_slave_id()->MergeFrom(task->slave_id());
- TaskStatus* status = update->mutable_status();
- status->mutable_task_id()->MergeFrom(task->task_id());
- status->set_state(TASK_LOST);
- status->set_message("Slave " + slave->info.hostname() + " disconnected");
- update->set_timestamp(Clock::now().secs());
- update->set_uuid(UUID::random().toBytes());
send(framework->pid, message);
// Remove the task from slave and framework.
http://git-wip-us.apache.org/repos/asf/mesos/blob/bf703d2a/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index 0315c52..2480b5c 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -799,28 +799,48 @@ TYPED_TEST(SlaveRecoveryTest, RemoveNonCheckpointingFramework)
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
- TaskInfo task = createTask(offers.get()[0], "sleep 1000");
+ // Launch 2 tasks from this offer.
vector<TaskInfo> tasks;
- tasks.push_back(task); // Long-running task
+ Offer offer = offers.get()[0];
- Future<Nothing> update;
+ Offer offer1 = offer;
+ offer1.mutable_resources()->CopyFrom(Resources::parse("cpus:1;mem:512"));
+ tasks.push_back(createTask(offer1, "sleep 1000")); // Long-running task
+
+ Offer offer2 = offer;
+ offer2.mutable_resources()->CopyFrom(Resources::parse("cpus:1;mem:512"));
+ tasks.push_back(createTask(offer2, "sleep 1000")); // Long-running task
+
+ ASSERT_LE(Resources(offer1.resources()) + Resources(offer2.resources()),
+ Resources(offer.resources()));
+
+ Future<Nothing> update1;
+ Future<Nothing> update2;
EXPECT_CALL(sched, statusUpdate(_, _))
- .WillOnce(FutureSatisfy(&update));
+ .WillOnce(FutureSatisfy(&update1))
+ .WillOnce(FutureSatisfy(&update2));
driver.launchTasks(offers.get()[0].id(), tasks);
- // Wait for TASK_RUNNING update.
- AWAIT_READY(update);
+ // Wait for TASK_RUNNING updates from the tasks.
+ AWAIT_READY(update1);
+ AWAIT_READY(update2);
- Future<TaskStatus> status;
+ // The master should generate TASK_LOST updates once the slave is stopped.
+ Future<TaskStatus> status1;
+ Future<TaskStatus> status2;
EXPECT_CALL(sched, statusUpdate(_, _))
- .WillOnce(FutureArg<1>(&status));
+ .WillOnce(FutureArg<1>(&status1))
+ .WillOnce(FutureArg<1>(&status2));
this->Stop(slave.get());
- // Scheduler should receive the TASK_LOST update.
- AWAIT_READY(status);
- ASSERT_EQ(TASK_LOST, status.get().state());
+ // Scheduler should receive the TASK_LOST updates.
+ AWAIT_READY(status1);
+ ASSERT_EQ(TASK_LOST, status1.get().state());
+
+ AWAIT_READY(status2);
+ ASSERT_EQ(TASK_LOST, status2.get().state());
driver.stop();
driver.join();
[20/23] git commit: Fixed issues from cherry-picks for 0.13.0-rc4.
Posted by vi...@apache.org.
Fixed issues from cherry-picks for 0.13.0-rc4.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5c19b044
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5c19b044
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5c19b044
Branch: refs/heads/vinod/0.13.0
Commit: 5c19b044a6d8d7b0f24c219965b3b7ab8674d7ba
Parents: d6e399a
Author: Benjamin Mahler <bm...@twitter.com>
Authored: Fri Jul 12 16:36:24 2013 -0700
Committer: Benjamin Mahler <bm...@twitter.com>
Committed: Fri Jul 12 16:36:24 2013 -0700
----------------------------------------------------------------------
.../libprocess/3rdparty/stout/include/stout/stopwatch.hpp | 9 ++++++++-
3rdparty/libprocess/src/process.cpp | 10 +++-------
2 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5c19b044/3rdparty/libprocess/3rdparty/stout/include/stout/stopwatch.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/stout/stopwatch.hpp b/3rdparty/libprocess/3rdparty/stout/include/stout/stopwatch.hpp
index bcff8c6..97e3469 100644
--- a/3rdparty/libprocess/3rdparty/stout/include/stout/stopwatch.hpp
+++ b/3rdparty/libprocess/3rdparty/stout/include/stout/stopwatch.hpp
@@ -15,7 +15,14 @@
class Stopwatch
{
public:
- Stopwatch() : running(false) { started.tv_sec = 0; started.tv_nsec = 0; }
+ Stopwatch()
+ : running(false)
+ {
+ started.tv_sec = 0;
+ started.tv_nsec = 0;
+ stopped.tv_sec = 0;
+ stopped.tv_nsec = 0;
+ }
void start()
{
http://git-wip-us.apache.org/repos/asf/mesos/blob/5c19b044/3rdparty/libprocess/src/process.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/process.cpp b/3rdparty/libprocess/src/process.cpp
index 373097c..5ae3a80 100644
--- a/3rdparty/libprocess/src/process.cpp
+++ b/3rdparty/libprocess/src/process.cpp
@@ -301,10 +301,9 @@ class Route
{
public:
Route(const string& name,
- const Option<string>& help,
const lambda::function<Future<Response>(const Request&)>& handler)
{
- process = new RouteProcess(name, help, handler);
+ process = new RouteProcess(name, handler);
spawn(process);
}
@@ -320,16 +319,14 @@ private:
public:
RouteProcess(
const string& name,
- const Option<string>& _help,
const lambda::function<Future<Response>(const Request&)>& _handler)
: ProcessBase(strings::remove(name, "/", strings::PREFIX)),
- help(_help),
handler(_handler) {}
protected:
virtual void initialize()
{
- route("/", help, &RouteProcess::handle);
+ route("/", &RouteProcess::handle);
}
Future<Response> handle(const Request& request)
@@ -337,7 +334,6 @@ private:
return handler(request);
}
- const Option<string> help;
const lambda::function<Future<Response>(const Request&)> handler;
};
@@ -1475,7 +1471,7 @@ void initialize(const string& delegate)
lambda::function<Future<Response>(const Request&)> __processes__ =
lambda::bind(&ProcessManager::__processes__, process_manager, lambda::_1);
- new Route("/__processes__", None(), __processes__);
+ new Route("/__processes__", __processes__);
char temp[INET_ADDRSTRLEN];
if (inet_ntop(AF_INET, (in_addr*) &__ip__, temp, INET_ADDRSTRLEN) == NULL) {
[13/23] git commit: Changed slave command line flag 'safe' to
'strict'.
Posted by vi...@apache.org.
Changed slave command line flag 'safe' to 'strict'.
Review: https://reviews.apache.org/r/12084
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a6a2fe98
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a6a2fe98
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a6a2fe98
Branch: refs/heads/vinod/0.13.0
Commit: a6a2fe98f20ec2cf0ec6bd60dc7e300b0dbf9321
Parents: bf703d2
Author: Vinod Kone <vi...@twitter.com>
Authored: Tue Jun 25 12:13:43 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Mon Jul 1 12:47:20 2013 -0700
----------------------------------------------------------------------
src/slave/flags.hpp | 17 +++++++------
src/slave/slave.cpp | 8 +++---
src/slave/slave.hpp | 4 +--
src/slave/state.cpp | 44 ++++++++++++++++-----------------
src/slave/state.hpp | 16 ++++++------
src/tests/slave_recovery_tests.cpp | 2 +-
6 files changed, 46 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 9612983..94c393e 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -135,13 +135,14 @@ public:
" and the slave registers with the master as a new slave.",
"reconnect");
- add(&Flags::safe,
- "safe",
- "Whether to ignore (safe=false) any ambiguous errors during recovery.\n"
- "An ambiguous error is one where it is impossible to differentiate\n"
- "between a safe error that can be ignored and an unsafe error that "
- "cannot be.",
- true);
+ add(&Flags::strict,
+ "strict",
+ "If strict=true, any and all recovery errors are considered fatal.\n"
+ "If strict=false, any expected errors (e.g., slave cannot recover\n"
+ "information about an executor, because the slave died right before\n"
+ "the executor registered.) during recovery are ignored and as much\n"
+ "state as possible is recovered.\n",
+ false);
#ifdef __linux__
add(&Flags::cgroups_hierarchy,
@@ -182,7 +183,7 @@ public:
Duration resource_monitoring_interval;
bool checkpoint;
std::string recover;
- bool safe;
+ bool strict;
#ifdef __linux__
std::string cgroups_hierarchy;
std::string cgroups_root;
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index aada544..c56dafc 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -371,7 +371,7 @@ void Slave::initialize()
}
// Start recovery.
- recover(flags.recover == "reconnect", flags.safe)
+ recover(flags.recover == "reconnect", flags.strict)
.onAny(defer(self(), &Slave::_initialize, params::_1));
}
@@ -2512,7 +2512,7 @@ void Slave::_checkDiskUsage(const Future<Try<double> >& usage)
}
-Future<Nothing> Slave::recover(bool reconnect, bool safe)
+Future<Nothing> Slave::recover(bool reconnect, bool strict)
{
const string& metaDir = paths::getMetaRootDir(flags.work_dir);
@@ -2526,7 +2526,7 @@ Future<Nothing> Slave::recover(bool reconnect, bool safe)
}
// First, recover the slave state.
- Result<SlaveState> state = state::recover(metaDir, safe);
+ Result<SlaveState> state = state::recover(metaDir, strict);
if (state.isError()) {
EXIT(1) << "Failed to recover slave state: " << state.error();
}
@@ -2859,7 +2859,7 @@ Executor* Framework::recoverExecutor(const ExecutorState& state)
// Recover the libprocess PID if possible.
if (run.libprocessPid.isSome()) {
- // When recovering in unsafe mode, the assumption is that the
+ // When recovering in non-strict mode, the assumption is that the
// slave can die after checkpointing the forked pid but before the
// libprocess pid. So, it is not possible for the libprocess pid
// to exist but not the forked pid. If so, it is a really bad
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 3a147f0..a9fd9c9 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -238,8 +238,8 @@ protected:
// Reads the checkpointed data from a previous run and recovers state.
// If 'reconnect' is true, the slave attempts to reconnect to any old
// live executors. Otherwise, the slave attempts to shutdown/kill them.
- // If 'safe' is true, any recovery errors are considered fatal.
- Future<Nothing> recover(bool reconnect, bool safe);
+ // If 'strict' is true, any recovery errors are considered fatal.
+ Future<Nothing> recover(bool reconnect, bool strict);
Future<Nothing> _recover(const state::SlaveState& state, bool reconnect);
// Helper to recover a framework from the specified state.
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/slave/state.cpp
----------------------------------------------------------------------
diff --git a/src/slave/state.cpp b/src/slave/state.cpp
index 27a2731..e910ab7 100644
--- a/src/slave/state.cpp
+++ b/src/slave/state.cpp
@@ -28,7 +28,7 @@ using std::string;
using std::max;
-Result<SlaveState> recover(const string& rootDir, bool safe)
+Result<SlaveState> recover(const string& rootDir, bool strict)
{
LOG(INFO) << "Recovering state from " << rootDir;
@@ -51,7 +51,7 @@ Result<SlaveState> recover(const string& rootDir, bool safe)
SlaveID slaveId;
slaveId.set_value(os::basename(directory.get()).get());
- Try<SlaveState> state = SlaveState::recover(rootDir, slaveId, safe);
+ Try<SlaveState> state = SlaveState::recover(rootDir, slaveId, strict);
if (state.isError()) {
return Error(state.error());
}
@@ -63,7 +63,7 @@ Result<SlaveState> recover(const string& rootDir, bool safe)
Try<SlaveState> SlaveState::recover(
const string& rootDir,
const SlaveID& slaveId,
- bool safe)
+ bool strict)
{
SlaveState state;
state.id = slaveId;
@@ -75,7 +75,7 @@ Try<SlaveState> SlaveState::recover(
if (!slaveInfo.isSome()) {
const string& message = "Failed to read slave info from '" + path + "': " +
(slaveInfo.isError() ? slaveInfo.error() : " none");
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -100,7 +100,7 @@ Try<SlaveState> SlaveState::recover(
frameworkId.set_value(os::basename(path).get());
const Try<FrameworkState>& framework =
- FrameworkState::recover(rootDir, slaveId, frameworkId, safe);
+ FrameworkState::recover(rootDir, slaveId, frameworkId, strict);
if (framework.isError()) {
return Error("Failed to recover framework " + frameworkId.value() +
@@ -118,7 +118,7 @@ Try<FrameworkState> FrameworkState::recover(
const string& rootDir,
const SlaveID& slaveId,
const FrameworkID& frameworkId,
- bool safe)
+ bool strict)
{
FrameworkState state;
state.id = frameworkId;
@@ -133,7 +133,7 @@ Try<FrameworkState> FrameworkState::recover(
message = "Failed to read framework info from '" + path + "': " +
(frameworkInfo.isError() ? frameworkInfo.error() : " none");
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -151,7 +151,7 @@ Try<FrameworkState> FrameworkState::recover(
message =
"Failed to read framework pid from '" + path + "': " + pid.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -177,7 +177,7 @@ Try<FrameworkState> FrameworkState::recover(
executorId.set_value(os::basename(path).get());
const Try<ExecutorState>& executor =
- ExecutorState::recover(rootDir, slaveId, frameworkId, executorId, safe);
+ ExecutorState::recover(rootDir, slaveId, frameworkId, executorId, strict);
if (executor.isError()) {
return Error("Failed to recover executor " + executorId.value() +
@@ -196,7 +196,7 @@ Try<ExecutorState> ExecutorState::recover(
const SlaveID& slaveId,
const FrameworkID& frameworkId,
const ExecutorID& executorId,
- bool safe)
+ bool strict)
{
ExecutorState state;
state.id = executorId;
@@ -214,7 +214,7 @@ Try<ExecutorState> ExecutorState::recover(
"Failed to read executor info from '" + path +
"': " + (executorInfo.isError() ? executorInfo.error() : " none");
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -254,7 +254,7 @@ Try<ExecutorState> ExecutorState::recover(
const UUID& uuid = UUID::fromString(os::basename(path).get());
const Try<RunState>& run = RunState::recover(
- rootDir, slaveId, frameworkId, executorId, uuid, safe);
+ rootDir, slaveId, frameworkId, executorId, uuid, strict);
if (run.isError()) {
return Error("Failed to recover run " + uuid.toString() +
@@ -271,7 +271,7 @@ Try<ExecutorState> ExecutorState::recover(
message =
"Failed to find the latest run of executor '" + executorId.value() + "'";
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -289,7 +289,7 @@ Try<RunState> RunState::recover(
const FrameworkID& frameworkId,
const ExecutorID& executorId,
const UUID& uuid,
- bool safe)
+ bool strict)
{
RunState state;
state.id = uuid;
@@ -316,7 +316,7 @@ Try<RunState> RunState::recover(
taskId.set_value(os::basename(path).get());
const Try<TaskState>& task = TaskState::recover(
- rootDir, slaveId, frameworkId, executorId, uuid, taskId, safe);
+ rootDir, slaveId, frameworkId, executorId, uuid, taskId, strict);
if (task.isError()) {
return Error(
@@ -336,7 +336,7 @@ Try<RunState> RunState::recover(
message = "Failed to read executor's forked pid from '" + path +
"': " + pid.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -362,7 +362,7 @@ Try<RunState> RunState::recover(
message = "Failed to read executor's libprocess pid from '" + path +
"': " + pid.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -383,7 +383,7 @@ Try<TaskState> TaskState::recover(
const ExecutorID& executorId,
const UUID& uuid,
const TaskID& taskId,
- bool safe)
+ bool strict)
{
TaskState state;
state.id = taskId;
@@ -399,7 +399,7 @@ Try<TaskState> TaskState::recover(
message = "Failed to read task info from '" + path +
"': " + (task.isError() ? task.error() : " none");
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -420,7 +420,7 @@ Try<TaskState> TaskState::recover(
message = "Failed to open status updates file '" + path +
"': " + fd.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -449,7 +449,7 @@ Try<TaskState> TaskState::recover(
message = "Failed to read status updates file '" + path +
"': " + record.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
@@ -471,7 +471,7 @@ Try<TaskState> TaskState::recover(
message = "Failed to close status updates file '" + path +
"': " + close.error();
- if (safe) {
+ if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/slave/state.hpp
----------------------------------------------------------------------
diff --git a/src/slave/state.hpp b/src/slave/state.hpp
index 1e3e894..08e3617 100644
--- a/src/slave/state.hpp
+++ b/src/slave/state.hpp
@@ -52,10 +52,10 @@ struct RunState;
struct TaskState;
// Each of the structs below (recursively) recover the checkpointed
-// state. If the 'safe' flag is set, any errors encountered while
+// state. If the 'strict' flag is set, any errors encountered while
// recovering a state are considered fatal and hence the recovery is
// short-circuited and returns an error. There might be orphaned
-// executors that need to be manually cleaned up. If 'safe' flag is
+// executors that need to be manually cleaned up. If 'strict' flag is
// not set, any errors encountered are considered non-fatal and the
// recovery continues by recovering as much of the state as possible.
@@ -64,7 +64,7 @@ struct SlaveState
static Try<SlaveState> recover(
const std::string& rootDir,
const SlaveID& slaveId,
- bool safe);
+ bool strict);
SlaveID id;
Option<SlaveInfo> info;
@@ -78,7 +78,7 @@ struct FrameworkState
const std::string& rootDir,
const SlaveID& slaveId,
const FrameworkID& frameworkId,
- bool safe);
+ bool strict);
FrameworkID id;
Option<FrameworkInfo> info;
@@ -94,7 +94,7 @@ struct ExecutorState
const SlaveID& slaveId,
const FrameworkID& frameworkId,
const ExecutorID& executorId,
- bool safe);
+ bool strict);
ExecutorID id;
Option<ExecutorInfo> info;
@@ -111,7 +111,7 @@ struct RunState
const FrameworkID& frameworkId,
const ExecutorID& executorId,
const UUID& uuid,
- bool safe);
+ bool strict);
Option<UUID> id;
hashmap<TaskID, TaskState> tasks;
@@ -129,7 +129,7 @@ struct TaskState
const ExecutorID& executorId,
const UUID& uuid,
const TaskID& taskId,
- bool safe);
+ bool strict);
TaskID id;
Option<Task> info;
@@ -139,7 +139,7 @@ struct TaskState
// This function performs recovery from the state stored at 'rootDir'.
-Result<SlaveState> recover(const std::string& rootDir, bool safe);
+Result<SlaveState> recover(const std::string& rootDir, bool strict);
// Thin wrappers to checkpoint data to disk and perform the
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6a2fe98/src/tests/slave_recovery_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_recovery_tests.cpp b/src/tests/slave_recovery_tests.cpp
index 2480b5c..daae66e 100644
--- a/src/tests/slave_recovery_tests.cpp
+++ b/src/tests/slave_recovery_tests.cpp
@@ -126,7 +126,7 @@ public:
// Setup recovery slave flags.
flags.checkpoint = true;
flags.recover = "reconnect";
- flags.safe = false;
+ flags.strict = false;
return flags;
}
[02/23] git commit: Fixed slave to properly handle terminated tasks
that have pending updates.
Posted by vi...@apache.org.
Fixed slave to properly handle terminated tasks that have pending
updates.
Review: https://reviews.apache.org/r/11694
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/22e64506
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/22e64506
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/22e64506
Branch: refs/heads/vinod/0.13.0
Commit: 22e64506da12dc29ecd101bf73702c8287811bf2
Parents: 3f809e0
Author: Vinod Kone <vi...@twitter.com>
Authored: Wed Jun 5 22:47:57 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 15:19:28 2013 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 116 +++++++++++++++++++++++++++--------------------
src/slave/slave.hpp | 14 ++++--
2 files changed, 75 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/22e64506/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 8ce1646..840c64d 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -1041,21 +1041,14 @@ void Slave::killTask(const FrameworkID& frameworkId, const TaskID& taskId)
switch (executor->state) {
case Executor::REGISTERING: {
- if (executor->queuedTasks.contains(taskId)) {
- // We remove the task here so that if this executor registers at
- // a later point in time it won't be sent this task.
- LOG(WARNING) << "Removing queued task " << taskId
- << " from executor '" << executor->id
- << "' of framework " << frameworkId
- << " because the executor hasn't registered yet";
- executor->removeTask(taskId);
- } else {
- LOG(WARNING) << "Cannot kill task " << taskId
- << " of framework " << frameworkId
- << " because the executor '" << executor->id
- << "' hasn't registered yet";
- }
+ LOG(WARNING) << "Removing queued task " << taskId
+ << " of framework " << frameworkId
+ << " because the executor '" << executor->id
+ << "' hasn't registered yet";
+ // NOTE: Sending a TASK_KILLED update removes the task from
+ // Executor::queuedTasks, so that if the executor registers at
+ // a later point in time, it won't get this task.
const StatusUpdate& update = protobuf::createStatusUpdate(
frameworkId,
info.id(),
@@ -1376,11 +1369,13 @@ void Slave::_statusUpdateAcknowledgement(
executor->state == Executor::TERMINATED)
<< executor->state;
- executor->updates.remove(taskId, uuid);
+ if (executor->terminatedTasks.contains(taskId)) {
+ executor->completeTask(taskId);
+ }
// Remove the executor if it has terminated and there are no more
- // pending updates.
- if (executor->state == Executor::TERMINATED && executor->updates.empty()) {
+ // incomplete tasks.
+ if (executor->state == Executor::TERMINATED && !executor->incompleteTasks()) {
remove(framework, executor);
}
}
@@ -1594,15 +1589,13 @@ void Slave::reregisterExecutor(
send(executor->pid, message);
// Handle all the pending updates.
+ // The status update manager might have already checkpointed some
+ // of these pending updates (for example, if the slave died right
+ // after it checkpointed the update but before it could send the
+ // ACK to the executor). This is ok because the status update
+ // manager correctly handles duplicate updates.
foreach (const StatusUpdate& update, updates) {
- // The status update manager might have already checkpointed some
- // of these pending updates (for e.g: if the slave died right
- // after it checkpointed the update but before it could send the
- // ACK to the executor). If so, we can just ignore those updates.
- if (!executor->updates.contains(
- update.status().task_id(), UUID::fromBytes(update.uuid()))) {
- statusUpdate(update); // This also updates the executor's resources!
- }
+ statusUpdate(update); // This also updates the executor's resources!
}
// Now, if there is any task still in STAGING state and not in
@@ -1720,8 +1713,13 @@ void Slave::statusUpdate(const StatusUpdate& update)
if (executor == NULL) {
LOG(WARNING) << "Could not find the executor for "
<< "status update " << update;
- stats.invalidStatusUpdates++;
+ stats.validStatusUpdates++;
+ // NOTE: We forward the update here because this update could be
+ // generated by the slave when the executor is unknown to it,
+ // e.g., killTask(), _runTask().
+ // TODO(vinod): Revisit these semantics when we disallow updates
+ // sent by executors that are unknown to the slave.
statusUpdateManager->update(update, info.id())
.onAny(defer(self(), &Slave::_statusUpdate, params::_1, update, None()));
@@ -1740,11 +1738,10 @@ void Slave::statusUpdate(const StatusUpdate& update)
stats.validStatusUpdates++;
executor->updateTaskState(status.task_id(), status.state());
- executor->updates.put(status.task_id(), UUID::fromBytes(update.uuid()));
// Handle the task appropriately if it's terminated.
if (protobuf::isTerminalState(status.state())) {
- executor->removeTask(status.task_id());
+ executor->terminateTask(status.task_id(), status.state());
// Tell the isolator to update the resources.
dispatch(isolator,
@@ -2152,10 +2149,10 @@ void Slave::executorTerminated(
send(master, message);
}
- // Remove the executor if either there are no pending updates
- // or the framework is terminating.
- if (executor->updates.empty() ||
- framework->state == Framework::TERMINATING) {
+ // Remove the executor if either the framework is terminating or
+ // there are no incomplete tasks.
+ if (framework->state == Framework::TERMINATING ||
+ !executor->incompleteTasks()) {
remove(framework, executor);
}
break;
@@ -2186,8 +2183,8 @@ void Slave::remove(Framework* framework, Executor* executor)
// care for pending updates when a framework is terminating
// because the framework cannot ACK them.
CHECK(executor->state == Executor::TERMINATED) << executor->state;
- CHECK (executor->updates.empty() ||
- framework->state == Framework::TERMINATING);
+ CHECK(framework->state == Framework::TERMINATING ||
+ !executor->incompleteTasks());
// TODO(vinod): Move the responsibility of gc'ing to the
// Executor struct.
@@ -2789,7 +2786,7 @@ Executor* Framework::getExecutor(const TaskID& taskId)
foreachvalue (Executor* executor, executors) {
if (executor->queuedTasks.contains(taskId) ||
executor->launchedTasks.contains(taskId) ||
- executor->updates.contains(taskId)) {
+ executor->terminatedTasks.contains(taskId)) {
return executor;
}
}
@@ -2941,23 +2938,36 @@ Task* Executor::addTask(const TaskInfo& task)
}
-void Executor::removeTask(const TaskID& taskId)
+void Executor::terminateTask(
+ const TaskID& taskId,
+ const mesos::TaskState& state)
{
+ Task* task = NULL;
// Remove the task if it's queued.
- queuedTasks.erase(taskId);
-
- // Update the resources if it's been launched.
- if (launchedTasks.contains(taskId)) {
- Task* task = launchedTasks[taskId];
+ if (queuedTasks.contains(taskId)) {
+ task = new Task(
+ protobuf::createTask(queuedTasks[taskId], state, id, frameworkId));
+ } else if (launchedTasks.contains(taskId)) {
+ // Update the resources if it's been launched.
+ task = launchedTasks[taskId];
foreach (const Resource& resource, task->resources()) {
resources -= resource;
}
launchedTasks.erase(taskId);
+ }
- completedTasks.push_back(*task);
+ terminatedTasks[taskId] = CHECK_NOTNULL(task);
+}
- delete task;
- }
+
+void Executor::completeTask(const TaskID& taskId)
+{
+ CHECK(terminatedTasks.contains(taskId)) << "Unknown task " << taskId;
+
+ Task* task = terminatedTasks[taskId];
+ completedTasks.push_back(*task);
+ terminatedTasks.erase(taskId);
+ delete task;
}
@@ -3001,16 +3011,14 @@ void Executor::recoverTask(const TaskState& state)
// Read updates to get the latest state of the task.
foreach (const StatusUpdate& update, state.updates) {
updateTaskState(state.id, update.status().state());
- updates.put(state.id, UUID::fromBytes(update.uuid()));
- // Remove the task if it received a terminal update.
+ // Terminate the task if it received a terminal update.
if (protobuf::isTerminalState(update.status().state())) {
- removeTask(state.id);
+ terminateTask(state.id, update.status().state());
- // If the terminal update has been acknowledged, remove it
- // from pending tasks.
+ // If the terminal update has been acknowledged, remove it.
if (state.acks.contains(UUID::fromBytes(update.uuid()))) {
- updates.remove(state.id, UUID::fromBytes(update.uuid()));
+ completeTask(state.id);
}
break;
}
@@ -3026,6 +3034,14 @@ void Executor::updateTaskState(const TaskID& taskId, mesos::TaskState state)
}
+bool Executor::incompleteTasks()
+{
+ return !queuedTasks.empty() ||
+ !launchedTasks.empty() ||
+ !terminatedTasks.empty();
+}
+
+
std::ostream& operator << (std::ostream& stream, Framework::State state) {
switch (state) {
case Framework::RUNNING: return stream << "RUNNING";
http://git-wip-us.apache.org/repos/asf/mesos/blob/22e64506/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 26dc96e..5aba7ed 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -335,11 +335,15 @@ struct Executor
~Executor();
Task* addTask(const TaskInfo& task);
- void removeTask(const TaskID& taskId);
+ void terminateTask(const TaskID& taskId, const mesos::TaskState& state);
+ void completeTask(const TaskID& taskId);
void checkpointTask(const TaskInfo& task);
void recoverTask(const state::TaskState& state);
void updateTaskState(const TaskID& taskId, TaskState state);
+ // Returns true if there are any queued/launched/terminated tasks.
+ bool incompleteTasks();
+
enum State {
REGISTERING, // Executor is launched but not (re-)registered yet.
RUNNING, // Executor has (re-)registered.
@@ -367,13 +371,13 @@ struct Executor
Resources resources; // Currently consumed resources.
- hashmap<TaskID, TaskInfo> queuedTasks;
- hashmap<TaskID, Task*> launchedTasks;
+ hashmap<TaskID, TaskInfo> queuedTasks; // Not yet launched.
+ hashmap<TaskID, Task*> launchedTasks; // Running.
+ hashmap<TaskID, Task*> terminatedTasks; // Terminated but pending updates.
+ boost::circular_buffer<Task> completedTasks; // Terminated and updates acked.
multihashmap<TaskID, UUID> updates; // Pending updates.
- boost::circular_buffer<Task> completedTasks;
-
private:
Executor(const Executor&); // No copying.
Executor& operator = (const Executor&); // No assigning.
[07/23] git commit: Added a new 'statistics.json' endpoint to the
ResourceMonitor, this deprecates the old usage.json endpoint.
Posted by vi...@apache.org.
Added a new 'statistics.json' endpoint to the ResourceMonitor, this
deprecates the old usage.json endpoint.
This is v2 of the monitoring statistics, added with a new 'statistics.json' endpoint.
The 'usage.json' endpoint is now deprecated, and will be removed at a later point.
From: Ben Mahler <be...@gmail.com>
Review: https://reviews.apache.org/r/11641
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/062c850a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/062c850a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/062c850a
Branch: refs/heads/vinod/0.13.0
Commit: 062c850ae59a737bf2047a42e23012cdd1c84cee
Parents: 25f136a
Author: Vinod Kone <vi...@twitter.com>
Authored: Fri Jun 7 16:17:19 2013 -0700
Committer: Vinod Kone <vi...@twitter.com>
Committed: Fri Jun 7 16:18:05 2013 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 15 ++--
src/slave/cgroups_isolator.cpp | 21 ++++-
src/slave/cgroups_isolator.hpp | 2 +
src/slave/monitor.cpp | 155 +++++++++++++++++++++++++++++++-----
src/slave/monitor.hpp | 10 +++
src/slave/process_isolator.cpp | 51 +++++++++---
src/slave/process_isolator.hpp | 1 +
src/tests/isolator_tests.cpp | 21 +++--
src/tests/monitor_tests.cpp | 77 +++++++++++++++---
9 files changed, 294 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index ece6559..8cbcd9a 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -263,15 +263,18 @@ message ResourceStatistics {
required double timestamp = 1; // Snapshot time, in seconds since the Epoch.
// CPU Usage Information:
- // A percentage based cpu usage rate since the last snapshot.
- // This is akin to what the 'top' program shows.
- optional double cpu_usage = 2;
// Total CPU time spent in user mode, and kernel mode.
- optional double cpu_user_time = 3; // In seconds.
- optional double cpu_system_time = 4; // In seconds.
+ optional double cpus_user_time_secs = 2;
+ optional double cpus_system_time_secs = 3;
+
+ // Number of CPUs allocated.
+ required double cpus_limit = 4;
// Memory Usage Information:
- optional uint64 memory_rss = 5; // Resident Set Size (in bytes).
+ optional uint64 mem_rss_bytes = 5; // Resident Set Size.
+
+ // Amount of memory resources allocated.
+ optional uint64 mem_limit_bytes = 6;
// TODO(bmahler): Add disk usage.
// TODO(bmahler): Add network usage?
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/cgroups_isolator.cpp
----------------------------------------------------------------------
diff --git a/src/slave/cgroups_isolator.cpp b/src/slave/cgroups_isolator.cpp
index 9b3a3a5..553844b 100644
--- a/src/slave/cgroups_isolator.cpp
+++ b/src/slave/cgroups_isolator.cpp
@@ -651,6 +651,8 @@ void CgroupsIsolator::resourcesChanged(
return;
}
+ info->resources = resources;
+
LOG(INFO) << "Changing cgroup controls for executor " << executorId
<< " of framework " << frameworkId
<< " with resources " << resources;
@@ -689,6 +691,17 @@ Future<ResourceStatistics> CgroupsIsolator::usage(
ResourceStatistics result;
result.set_timestamp(Clock::now().secs());
+ // Set the resource allocations.
+ Option<Bytes> mem = info->resources.mem();
+ if (mem.isSome()) {
+ result.set_mem_limit_bytes(mem.get().bytes());
+ }
+
+ Option<double> cpus = info->resources.cpus();
+ if (cpus.isSome()) {
+ result.set_cpus_limit(cpus.get());
+ }
+
Try<hashmap<string, uint64_t> > stat =
cgroups::stat(hierarchy, info->name(), "cpuacct.stat");
@@ -700,8 +713,10 @@ Future<ResourceStatistics> CgroupsIsolator::usage(
// TODO(bmahler): Add namespacing to cgroups to enforce the expected
// structure, e.g., cgroups::cpuacct::stat.
if (stat.get().contains("user") && stat.get().contains("system")) {
- result.set_cpu_user_time((double) stat.get()["user"] / (double) ticks);
- result.set_cpu_system_time((double) stat.get()["system"] / (double) ticks);
+ result.set_cpus_user_time_secs(
+ (double) stat.get()["user"] / (double) ticks);
+ result.set_cpus_system_time_secs(
+ (double) stat.get()["system"] / (double) ticks);
}
stat = cgroups::stat(hierarchy, info->name(), "memory.stat");
@@ -714,7 +729,7 @@ Future<ResourceStatistics> CgroupsIsolator::usage(
// TODO(bmahler): Add namespacing to cgroups to enforce the expected
// structure, e.g, cgroups::memory::stat.
if (stat.get().contains("rss")) {
- result.set_memory_rss(stat.get()["rss"]);
+ result.set_mem_rss_bytes(stat.get()["rss"]);
}
return result;
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/cgroups_isolator.hpp
----------------------------------------------------------------------
diff --git a/src/slave/cgroups_isolator.hpp b/src/slave/cgroups_isolator.hpp
index 7b8270d..124a4b3 100644
--- a/src/slave/cgroups_isolator.hpp
+++ b/src/slave/cgroups_isolator.hpp
@@ -169,6 +169,8 @@ private:
Flags flags; // Slave flags.
+ Resources resources; // Resources allocated to the cgroup.
+
// Used to cancel the OOM listening.
process::Future<uint64_t> oomNotifier;
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/monitor.cpp
----------------------------------------------------------------------
diff --git a/src/slave/monitor.cpp b/src/slave/monitor.cpp
index 5de1c15..4f3c91f 100644
--- a/src/slave/monitor.cpp
+++ b/src/slave/monitor.cpp
@@ -37,6 +37,8 @@
using namespace process;
+using process::statistics;
+
using std::map;
using std::string;
@@ -49,7 +51,15 @@ using process::wait; // Necessary on some OS's to disambiguate.
// Resource statistics constants.
// These match the names in the ResourceStatistics protobuf.
// TODO(bmahler): Later, when we have a richer monitoring story,
-// we will want to publish these outisde of this file.
+// we will want to publish these outside of this file.
+const std::string CPUS_TIME_SECS = "cpus_time_secs";
+const std::string CPUS_USER_TIME_SECS = "cpus_user_time_secs";
+const std::string CPUS_SYSTEM_TIME_SECS = "cpus_system_time_secs";
+const std::string CPUS_LIMIT = "cpus_limit";
+const std::string MEM_RSS_BYTES = "mem_rss_bytes";
+const std::string MEM_LIMIT_BYTES = "mem_limit_bytes";
+
+// TODO(bmahler): Deprecated statistical names, these will be removed!
const std::string CPU_TIME = "cpu_time";
const std::string CPU_USAGE = "cpu_usage";
const std::string MEMORY_RSS = "memory_rss";
@@ -61,11 +71,15 @@ void publish(
const ExecutorID& executorId,
const ResourceStatistics& statistics);
-Future<http::Response> _usage(
+Future<http::Response> _statisticsJSON(
const hashmap<FrameworkID, hashmap<ExecutorID, ExecutorInfo> >& executors,
const map<string, double>& statistics,
const Option<string>& jsonp);
+Future<http::Response> _usage(
+ const hashmap<FrameworkID, hashmap<ExecutorID, ExecutorInfo> >& executors,
+ const map<string, double>& statistics,
+ const Option<string>& jsonp);
Future<Nothing> ResourceMonitorProcess::watch(
const FrameworkID& frameworkId,
@@ -84,9 +98,9 @@ Future<Nothing> ResourceMonitorProcess::watch(
const string& prefix =
strings::join("/", frameworkId.value(), executorId.value(), "");
- process::statistics->meter(
+ ::statistics->meter(
"monitor",
- prefix + CPU_TIME,
+ prefix + CPUS_TIME_SECS,
new meters::TimeRate(prefix + CPU_USAGE));
// Schedule the resource collection.
@@ -105,8 +119,13 @@ Future<Nothing> ResourceMonitorProcess::unwatch(
// In case we've already noticed the executor was terminated,
// we need to archive the statistics first.
- process::statistics->archive("monitor", prefix + MEMORY_RSS);
- process::statistics->archive("monitor", prefix + CPU_TIME);
+ // No need to archive CPUS_USAGE as it is implicitly archived along
+ // with CPUS_TIME_SECS.
+ ::statistics->archive("monitor", prefix + CPUS_USER_TIME_SECS);
+ ::statistics->archive("monitor", prefix + CPUS_SYSTEM_TIME_SECS);
+ ::statistics->archive("monitor", prefix + CPUS_LIMIT);
+ ::statistics->archive("monitor", prefix + MEM_RSS_BYTES);
+ ::statistics->archive("monitor", prefix + MEM_LIMIT_BYTES);
if (!watches.contains(frameworkId) ||
!watches[frameworkId].contains(executorId)) {
@@ -195,20 +214,114 @@ void publish(
const string& prefix =
strings::join("/", frameworkId.value(), executorId.value(), "");
- // Publish memory statistic.
- process::statistics->set(
+ // Publish cpu usage statistics.
+ ::statistics->set(
+ "monitor",
+ prefix + CPUS_USER_TIME_SECS,
+ statistics.cpus_user_time_secs(),
+ time);
+ ::statistics->set(
"monitor",
- prefix + MEMORY_RSS,
- statistics.memory_rss(),
+ prefix + CPUS_SYSTEM_TIME_SECS,
+ statistics.cpus_system_time_secs(),
+ time);
+ ::statistics->set(
+ "monitor",
+ prefix + CPUS_LIMIT,
+ statistics.cpus_limit(),
+ time);
+ // The applied meter from watch() will publish the cpu usage.
+ ::statistics->set(
+ "monitor",
+ prefix + CPUS_TIME_SECS,
+ statistics.cpus_user_time_secs() + statistics.cpus_system_time_secs(),
time);
- // Publish cpu usage statistics. The applied meter from watch()
- // will publish the cpu usage percentage.
- process::statistics->set(
+ // Publish memory statistics.
+ ::statistics->set(
"monitor",
- prefix + CPU_TIME,
- statistics.cpu_user_time() + statistics.cpu_system_time(),
+ prefix + MEM_RSS_BYTES,
+ statistics.mem_rss_bytes(),
time);
+ ::statistics->set(
+ "monitor",
+ prefix + MEM_LIMIT_BYTES,
+ statistics.mem_limit_bytes(),
+ time);
+}
+
+
+Future<http::Response> ResourceMonitorProcess::statisticsJSON(
+ const http::Request& request)
+{
+ lambda::function<Future<http::Response>(const map<string, double>&)>
+ _statisticsJSON = lambda::bind(
+ slave::_statisticsJSON,
+ watches,
+ lambda::_1,
+ request.query.get("jsonp"));
+
+ return ::statistics->get("monitor").then(_statisticsJSON);
+}
+
+
+Future<http::Response> _statisticsJSON(
+ const hashmap<FrameworkID, hashmap<ExecutorID, ExecutorInfo> >& watches,
+ const map<string, double>& statistics,
+ const Option<string>& jsonp)
+{
+ JSON::Array result;
+
+ foreachkey (const FrameworkID& frameworkId, watches) {
+ foreachkey (const ExecutorID& executorId, watches.get(frameworkId).get()) {
+ const ExecutorInfo& info =
+ watches.get(frameworkId).get().get(executorId).get();
+ const string& prefix =
+ strings::join("/", frameworkId.value(), executorId.value(), "");
+
+ // Export zero values by default.
+ JSON::Object usage;
+ usage.values[CPUS_USER_TIME_SECS] = 0;
+ usage.values[CPUS_SYSTEM_TIME_SECS] = 0;
+ usage.values[CPUS_LIMIT] = 0;
+ usage.values[MEM_RSS_BYTES] = 0;
+ usage.values[MEM_LIMIT_BYTES] = 0;
+
+ // Set the cpu usage data if present.
+ if (statistics.count(prefix + CPUS_USER_TIME_SECS) > 0) {
+ usage.values[CPUS_USER_TIME_SECS] =
+ statistics.find(prefix + CPUS_USER_TIME_SECS)->second;
+ }
+ if (statistics.count(prefix + CPUS_SYSTEM_TIME_SECS) > 0) {
+ usage.values[CPUS_SYSTEM_TIME_SECS] =
+ statistics.find(prefix + CPUS_SYSTEM_TIME_SECS)->second;
+ }
+ if (statistics.count(prefix + CPUS_LIMIT) > 0) {
+ usage.values[CPUS_LIMIT] = statistics.find(prefix + CPUS_LIMIT)->second;
+ }
+
+ // Set the memory usage data if present.
+ if (statistics.count(prefix + MEM_RSS_BYTES) > 0) {
+ usage.values[MEM_RSS_BYTES] =
+ statistics.find(prefix + MEM_RSS_BYTES)->second;
+ }
+ if (statistics.count(prefix + MEM_LIMIT_BYTES) > 0) {
+ usage.values[MEM_LIMIT_BYTES] =
+ statistics.find(prefix + MEM_LIMIT_BYTES)->second;
+ }
+
+ JSON::Object entry;
+ entry.values["framework_id"] = frameworkId.value();
+ entry.values["executor_id"] = executorId.value();
+ entry.values["executor_name"] = info.name();
+ entry.values["source"] = info.source();
+ entry.values["statistics"] = usage;
+
+ result.values.push_back(entry);
+ }
+ }
+
+ return http::OK(result, jsonp);
}
@@ -222,7 +335,7 @@ Future<http::Response> ResourceMonitorProcess::usage(
lambda::_1,
request.query.get("jsonp"));
- return process::statistics->get("monitor").then(_usage);
+ return ::statistics->get("monitor").then(_usage);
}
@@ -250,11 +363,13 @@ Future<http::Response> _usage(
if (statistics.count(prefix + CPU_USAGE) > 0) {
usage.values[CPU_USAGE] = statistics.find(prefix + CPU_USAGE)->second;
}
- if (statistics.count(prefix + CPU_TIME) > 0) {
- usage.values[CPU_TIME] = statistics.find(prefix + CPU_TIME)->second;
+ if (statistics.count(prefix + CPUS_TIME_SECS) > 0) {
+ usage.values[CPU_TIME] =
+ statistics.find(prefix + CPUS_TIME_SECS)->second;
}
- if (statistics.count(prefix + MEMORY_RSS) > 0) {
- usage.values[MEMORY_RSS] = statistics.find(prefix + MEMORY_RSS)->second;
+ if (statistics.count(prefix + MEM_RSS_BYTES) > 0) {
+ usage.values[MEMORY_RSS] =
+ statistics.find(prefix + MEM_RSS_BYTES)->second;
}
JSON::Object entry;
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/monitor.hpp
----------------------------------------------------------------------
diff --git a/src/slave/monitor.hpp b/src/slave/monitor.hpp
index 8770dc0..e5fdcbd 100644
--- a/src/slave/monitor.hpp
+++ b/src/slave/monitor.hpp
@@ -44,6 +44,10 @@ class ResourceMonitorProcess;
// Provides resource monitoring for executors. Resource usage time
// series are stored using the Statistics module. Usage information
// is also exported via a JSON endpoint.
+// TODO(bmahler): Once the deprecated usage.json endpoint is removed,
+// clean this up!! It will be possible to drive collection directly
+// via the http endpoints, and the isolator can return all
+// information in one request.
// TODO(bmahler): Forward usage information to the master.
// TODO(bmahler): Consider pulling out the resource collection into
// a Collector abstraction. The monitor can then become a true
@@ -98,6 +102,7 @@ public:
protected:
virtual void initialize()
{
+ route("/statistics.json", &ResourceMonitorProcess::statisticsJSON);
route("/usage.json", &ResourceMonitorProcess::usage);
}
@@ -113,6 +118,11 @@ private:
const ExecutorID& executorId,
const Duration& interval);
+ // Returns the monitoring statistics. Requests have no parameters.
+ process::Future<process::http::Response> statisticsJSON(
+ const process::http::Request& request);
+
+ // TODO(bmahler): Deprecated.
// Returns the usage information. Requests have no parameters.
process::Future<process::http::Response> usage(
const process::http::Request& request);
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/process_isolator.cpp
----------------------------------------------------------------------
diff --git a/src/slave/process_isolator.cpp b/src/slave/process_isolator.cpp
index d4f7b76..b54bf7e 100644
--- a/src/slave/process_isolator.cpp
+++ b/src/slave/process_isolator.cpp
@@ -296,6 +296,19 @@ void ProcessIsolator::resourcesChanged(
const Resources& resources)
{
CHECK(initialized) << "Cannot do resourcesChanged before initialization!";
+
+ if (!infos.contains(frameworkId) ||
+ !infos[frameworkId].contains(executorId) ||
+ infos[frameworkId][executorId]->killed) {
+ LOG(INFO) << "Asked to update resources for an unknown/killed executor '"
+ << executorId << "' of framework " << frameworkId;
+ return;
+ }
+
+ ProcessInfo* info = CHECK_NOTNULL(infos[frameworkId][executorId]);
+
+ info->resources = resources;
+
// Do nothing; subclasses may override this.
}
@@ -365,6 +378,17 @@ Future<ResourceStatistics> ProcessIsolator::usage(
result.set_timestamp(Clock::now().secs());
+ // Set the resource allocations.
+ Option<Bytes> mem = info->resources.mem();
+ if (mem.isSome()) {
+ result.set_mem_limit_bytes(mem.get().bytes());
+ }
+
+ Option<double> cpus = info->resources.cpus();
+ if (cpus.isSome()) {
+ result.set_cpus_limit(cpus.get());
+ }
+
#ifdef __linux__
// Get the page size, used for memory accounting.
// NOTE: This is more portable than using getpagesize().
@@ -384,9 +408,11 @@ Future<ResourceStatistics> ProcessIsolator::usage(
return Future<ResourceStatistics>::failed(status.error());
}
- result.set_memory_rss(status.get().rss * pageSize);
- result.set_cpu_user_time((double) status.get().utime / (double) ticks);
- result.set_cpu_system_time((double) status.get().stime / (double) ticks);
+ result.set_mem_rss_bytes(status.get().rss * pageSize);
+ result.set_cpus_user_time_secs(
+ (double) status.get().utime / (double) ticks);
+ result.set_cpus_system_time_secs(
+ (double) status.get().stime / (double) ticks);
// Now aggregate all descendant process usage statistics.
Try<set<pid_t> > children = proc::children(info->pid.get(), true);
@@ -408,16 +434,15 @@ Future<ResourceStatistics> ProcessIsolator::usage(
continue;
}
- result.set_memory_rss(
- result.memory_rss() +
- status.get().rss * pageSize);
+ result.set_mem_rss_bytes(
+ result.mem_rss_bytes() + status.get().rss * pageSize);
- result.set_cpu_user_time(
- result.cpu_user_time() +
+ result.set_cpus_user_time_secs(
+ result.cpus_user_time_secs() +
(double) status.get().utime / (double) ticks);
- result.set_cpu_system_time(
- result.cpu_system_time() +
+ result.set_cpus_system_time_secs(
+ result.cpus_system_time_secs() +
(double) status.get().stime / (double) ticks);
}
#elif defined __APPLE__
@@ -441,11 +466,11 @@ Future<ResourceStatistics> ProcessIsolator::usage(
"Failed to get proc_pidinfo: " + stringify(size));
}
- result.set_memory_rss(task.pti_resident_size);
+ result.set_mem_rss_bytes(task.pti_resident_size);
// NOTE: CPU Times are in nanoseconds, but this is not documented!
- result.set_cpu_user_time(Nanoseconds(task.pti_total_user).secs());
- result.set_cpu_system_time(Nanoseconds(task.pti_total_system).secs());
+ result.set_cpus_user_time_secs(Nanoseconds(task.pti_total_user).secs());
+ result.set_cpus_system_time_secs(Nanoseconds(task.pti_total_system).secs());
#endif
return result;
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/slave/process_isolator.hpp
----------------------------------------------------------------------
diff --git a/src/slave/process_isolator.hpp b/src/slave/process_isolator.hpp
index 9875f4a..ee693f6 100644
--- a/src/slave/process_isolator.hpp
+++ b/src/slave/process_isolator.hpp
@@ -101,6 +101,7 @@ private:
ExecutorID executorId;
Option<pid_t> pid; // PID of the forked executor process.
bool killed; // True if "killing" has been initiated via 'killExecutor'.
+ Resources resources; // Resources allocated to the process tree.
};
// TODO(benh): Make variables const by passing them via constructor.
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/tests/isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/isolator_tests.cpp b/src/tests/isolator_tests.cpp
index aae8b2f..7013fa2 100644
--- a/src/tests/isolator_tests.cpp
+++ b/src/tests/isolator_tests.cpp
@@ -119,6 +119,12 @@ TYPED_TEST(IsolatorTest, Usage)
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
+ Resources resources(offers.get()[0].resources());
+ Option<Bytes> mem = resources.mem();
+ ASSERT_SOME(mem);
+ Option<double> cpus = resources.cpus();
+ ASSERT_SOME(cpus);
+
const std::string& file = path::join(flags.work_dir, "ready");
// This task induces user/system load in a child process by
@@ -172,9 +178,9 @@ TYPED_TEST(IsolatorTest, Usage)
statistics = usage.get();
// If we meet our usage expectations, we're done!
- if (statistics.memory_rss() >= 1024u &&
- statistics.cpu_user_time() >= 0.125 &&
- statistics.cpu_system_time() >= 0.125) {
+ if (statistics.cpus_user_time_secs() >= 0.125 &&
+ statistics.cpus_system_time_secs() >= 0.125 &&
+ statistics.mem_rss_bytes() >= 1024u) {
break;
}
@@ -182,9 +188,12 @@ TYPED_TEST(IsolatorTest, Usage)
waited += Milliseconds(100);
} while (waited < Seconds(10));
- EXPECT_GE(statistics.memory_rss(), 1024u);
- EXPECT_GE(statistics.cpu_user_time(), 0.125);
- EXPECT_GE(statistics.cpu_system_time(), 0.125);
+
+ EXPECT_GE(statistics.cpus_user_time_secs(), 0.125);
+ EXPECT_GE(statistics.cpus_system_time_secs(), 0.125);
+ EXPECT_EQ(statistics.cpus_limit(), cpus.get());
+ EXPECT_GE(statistics.mem_rss_bytes(), 1024u);
+ EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes());
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
http://git-wip-us.apache.org/repos/asf/mesos/blob/062c850a/src/tests/monitor_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/monitor_tests.cpp b/src/tests/monitor_tests.cpp
index 53920a0..3142416 100644
--- a/src/tests/monitor_tests.cpp
+++ b/src/tests/monitor_tests.cpp
@@ -72,21 +72,34 @@ TEST(MonitorTest, WatchUnwatch)
executorInfo.set_name("name");
executorInfo.set_source("source");
+ ResourceStatistics initialStatistics;
+ initialStatistics.set_cpus_user_time_secs(0);
+ initialStatistics.set_cpus_system_time_secs(0);
+ initialStatistics.set_cpus_limit(1.0);
+ initialStatistics.set_mem_rss_bytes(0);
+ initialStatistics.set_mem_limit_bytes(2048);
+ initialStatistics.set_timestamp(Clock::now().secs());
+
ResourceStatistics statistics;
- statistics.set_cpu_user_time(5);
- statistics.set_cpu_system_time(1);
- statistics.set_memory_rss(1024);
- statistics.set_timestamp(Clock::now().secs());
+ statistics.set_cpus_user_time_secs(4);
+ statistics.set_cpus_system_time_secs(1);
+ statistics.set_cpus_limit(1.0);
+ statistics.set_mem_rss_bytes(1024);
+ statistics.set_mem_limit_bytes(2048);
+ statistics.set_timestamp(
+ initialStatistics.timestamp() +
+ slave::RESOURCE_MONITORING_INTERVAL.secs());
TestingIsolator isolator;
process::spawn(isolator);
- Future<Nothing> usage;
+ Future<Nothing> usage1, usage2;
EXPECT_CALL(isolator, usage(frameworkId, executorId))
- .WillOnce(DoAll(FutureSatisfy(&usage),
+ .WillOnce(DoAll(FutureSatisfy(&usage1),
+ Return(initialStatistics)))
+ .WillOnce(DoAll(FutureSatisfy(&usage2),
Return(statistics)));
-
slave::ResourceMonitor monitor(&isolator);
// We pause the clock first in order to make sure that we can
@@ -107,7 +120,16 @@ TEST(MonitorTest, WatchUnwatch)
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
- AWAIT_READY(usage);
+ AWAIT_READY(usage1);
+
+ // Wait until the isolator has finished returning the statistics.
+ process::Clock::settle();
+
+ // The second collection will populate the cpus_usage.
+ process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
+ process::Clock::settle();
+
+ AWAIT_READY(usage2);
// Wait until the isolator has finished returning the statistics.
process::Clock::settle();
@@ -136,9 +158,42 @@ TEST(MonitorTest, WatchUnwatch)
"},"
"\"source\":\"source\""
"}]",
- statistics.cpu_user_time() + statistics.cpu_system_time(),
- statistics.cpu_usage(),
- statistics.memory_rss()).get(),
+ statistics.cpus_system_time_secs() + statistics.cpus_user_time_secs(),
+ (statistics.cpus_system_time_secs() +
+ statistics.cpus_user_time_secs()) /
+ slave::RESOURCE_MONITORING_INTERVAL.secs(),
+ statistics.mem_rss_bytes()).get(),
+ response);
+
+ response = process::http::get(upid, "statistics.json");
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(
+ "application/json",
+ "Content-Type",
+ response);
+
+ // TODO(bmahler): Verify metering directly through statistics.
+ AWAIT_EXPECT_RESPONSE_BODY_EQ(
+ strings::format(
+ "[{"
+ "\"executor_id\":\"executor\","
+ "\"executor_name\":\"name\","
+ "\"framework_id\":\"framework\","
+ "\"source\":\"source\","
+ "\"statistics\":{"
+ "\"cpus_limit\":%g,"
+ "\"cpus_system_time_secs\":%g,"
+ "\"cpus_user_time_secs\":%g,"
+ "\"mem_limit_bytes\":%lu,"
+ "\"mem_rss_bytes\":%lu"
+ "}"
+ "}]",
+ statistics.cpus_limit(),
+ statistics.cpus_system_time_secs(),
+ statistics.cpus_user_time_secs(),
+ statistics.mem_limit_bytes(),
+ statistics.mem_rss_bytes()).get(),
response);
// Ensure the monitor stops polling the isolator.