You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2016/04/13 00:34:40 UTC
[1/2] mesos git commit: Added labels to ExecutorInfo.
Repository: mesos
Updated Branches:
refs/heads/master 2e76199a3 -> c7802c030
Added labels to ExecutorInfo.
Review: https://reviews.apache.org/r/45572/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c7802c03
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c7802c03
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c7802c03
Branch: refs/heads/master
Commit: c7802c030d0cdfed1fd330262a26e07d87f274f2
Parents: f63b4ee
Author: Zhitao Li <zh...@gmail.com>
Authored: Tue Apr 12 11:54:07 2016 -0700
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Tue Apr 12 15:31:02 2016 -0700
----------------------------------------------------------------------
CHANGELOG | 6 ++
include/mesos/mesos.proto | 13 ++++-
include/mesos/v1/mesos.proto | 13 ++++-
src/common/http.cpp | 9 +++
src/slave/http.cpp | 4 ++
src/tests/master_tests.cpp | 94 +++++++++++++++++++++++++++++++
src/tests/oversubscription_tests.cpp | 10 ++++
src/tests/slave_tests.cpp | 94 +++++++++++++++++++++++++++++++
8 files changed, 241 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 5de6825..cbc4f9b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,7 @@
Release Notes - Mesos - Version 0.29.0 (WIP)
--------------------------------------------
This release contains the following new features:
+
* [MESOS-4909] - Tasks can now specify a kill policy. They are best-effort,
because machine failures or forcible terminations may occur. Currently, the
only available kill policy is how long to wait between graceful and forcible
@@ -40,6 +41,9 @@ Deprecations:
* [MESOS-5001] - The 'allocator/event_queue_dispatches' metric is now
deprecated in favor 'of allocator/mesos/event_queue_dispatches'.
+ * [MESOS-5029] - Deprecated the ExecutorInfo.source field in favor of
+ ExecutorInfo.labels.
+
Additional API Changes:
* [MESOS-4580] - Returning `202` (Accepted) for /reserve and related endpoints.
@@ -52,6 +56,8 @@ Additional API Changes:
* [MESOS-5015] - Changes Call and Event Type enums in executor.proto
from required to optional for the purpose of backwards compatibility.
+ * [MESOS-5029] - Added 'labels' to ExecutorInfo.
+
3rd Party Upgrades:
* [MESOS-4805] - Upgraded vendored ry-http-parser-1c3624a to nodejs/http-parser 2.6.1.
* [MESOS-4678] - Upgraded vendored protobuf 2.5.0 to 2.6.1.
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 63c181a..eefa292 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -472,7 +472,10 @@ message ExecutorInfo {
// NOTE: 'source' is exposed alongside the resource usage of the
// executor via JSON on the slave. This allows users to import usage
// information into a time series database for monitoring.
- optional string source = 10;
+ //
+ // This field is deprecated since 0.29.0. Please use labels for
+ // free-form metadata instead.
+ optional string source = 10 [deprecated = true]; // Since 0.29.0.
optional bytes data = 4;
@@ -489,6 +492,14 @@ message ExecutorInfo {
// grace period, as the agent may decide to allot a shorter
// period and failures / forcible terminations may occur.
optional DurationInfo shutdown_grace_period = 13;
+
+ // Labels are free-form key value pairs which are exposed through
+ // master and slave endpoints. Labels will not be interpreted or
+ // acted upon by Mesos itself. As opposed to the data field, labels
+ // will be kept in memory on master and slave processes. Therefore,
+ // labels should be used to tag executors with lightweight metadata.
+ // Labels should not contain duplicate key-value pairs.
+ optional Labels labels = 14;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index a60a834..9b4f68f 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -472,7 +472,10 @@ message ExecutorInfo {
// NOTE: 'source' is exposed alongside the resource usage of the
// executor via JSON on the agent. This allows users to import usage
// information into a time series database for monitoring.
- optional string source = 10;
+ //
+ // This field is deprecated since 0.29.0. Please use labels for
+ // free-form metadata instead.
+ optional string source = 10 [deprecated = true]; // Since 0.29.0.
optional bytes data = 4;
@@ -489,6 +492,14 @@ message ExecutorInfo {
// grace period, as the agent may decide to allot a shorter
// period and failures / forcible terminations may occur.
optional DurationInfo shutdown_grace_period = 13;
+
+ // Labels are free-form key value pairs which are exposed through
+ // master and agent endpoints. Labels will not be interpreted or
+ // acted upon by Mesos itself. As opposed to the data field, labels
+ // will be kept in memory on master and agent processes. Therefore,
+ // labels should be used to tag executors with lightweight metadata.
+ // Labels should not contain duplicate key-value pairs.
+ optional Labels labels = 14;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/src/common/http.cpp
----------------------------------------------------------------------
diff --git a/src/common/http.cpp b/src/common/http.cpp
index 735796c..3a8e9ee 100644
--- a/src/common/http.cpp
+++ b/src/common/http.cpp
@@ -354,6 +354,11 @@ JSON::Object model(const ExecutorInfo& executorInfo)
object.values["framework_id"] = executorInfo.framework_id().value();
object.values["command"] = model(executorInfo.command());
object.values["resources"] = model(executorInfo.resources());
+
+ if (executorInfo.has_labels()) {
+ object.values["labels"] = model(executorInfo.labels());
+ }
+
return object;
}
@@ -453,6 +458,10 @@ void json(JSON::ObjectWriter* writer, const ExecutorInfo& executorInfo)
writer->field("framework_id", executorInfo.framework_id().value());
writer->field("command", executorInfo.command());
writer->field("resources", Resources(executorInfo.resources()));
+
+ if (executorInfo.has_labels()) {
+ writer->field("labels", executorInfo.labels());
+ }
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index a684ff5..922aaad 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -114,6 +114,10 @@ static void json(JSON::ObjectWriter* writer, const Executor& executor)
writer->field("directory", executor.directory);
writer->field("resources", executor.resources);
+ if (executor.info.has_labels()) {
+ writer->field("labels", executor.info.labels());
+ }
+
writer->field("tasks", [&executor](JSON::ArrayWriter* writer) {
foreach (Task* task, executor.launchedTasks.values()) {
writer->element(*task);
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 806a958..1ae7260 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -3168,6 +3168,100 @@ TEST_F(MasterTest, StateSummaryEndpoint)
}
+// This test verifies that executor labels are
+// exposed in the master's state endpoint.
+TEST_F(MasterTest, ExecutorLabels)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+ TestContainerizer containerizer(&exec);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(1);
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
+ task.mutable_resources()->MergeFrom(offers.get()[0].resources());
+ task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
+
+ // Add three labels to the executor, two of which shares the same key.
+ Labels* labels = task.mutable_executor()->mutable_labels();
+
+ labels->add_labels()->CopyFrom(createLabel("key1", "value1"));
+ labels->add_labels()->CopyFrom(createLabel("key2", "value2"));
+ labels->add_labels()->CopyFrom(createLabel("key1", "value3"));
+
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .Times(1);
+
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY(status);
+ EXPECT_EQ(TASK_RUNNING, status.get().state());
+
+ // Verify label key and value in the master's state endpoint.
+ Future<Response> response = process::http::get(
+ master.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response->body);
+ ASSERT_SOME(parse);
+
+ Result<JSON::Array> labels_ = parse->find<JSON::Array>(
+ "frameworks[0].executors[0].labels");
+ EXPECT_SOME(labels_);
+
+ // Verify the contents of labels.
+ EXPECT_EQ(3u, labels_->values.size());
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key1", "value1"))),
+ labels_->values[0]);
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key2", "value2"))),
+ labels_->values[1]);
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key1", "value3"))),
+ labels_->values[2]);
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test verifies that label values are exposed over the master's
// state endpoint.
TEST_F(MasterTest, TaskLabels)
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/src/tests/oversubscription_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/oversubscription_tests.cpp b/src/tests/oversubscription_tests.cpp
index 07c6658..f983765 100644
--- a/src/tests/oversubscription_tests.cpp
+++ b/src/tests/oversubscription_tests.cpp
@@ -55,6 +55,8 @@ using namespace process;
using mesos::internal::master::Master;
+using mesos::internal::protobuf::createLabel;
+
using mesos::internal::slave::LoadQoSController;
using mesos::internal::slave::Slave;
@@ -225,6 +227,8 @@ TEST_F(OversubscriptionTest, FetchResourceUsage)
EXPECT_NE(0u, offers.get().size());
TaskInfo task = createTask(offers.get()[0], "sleep 10", DEFAULT_EXECUTOR_ID);
+ task.mutable_executor()->mutable_labels()->add_labels()->CopyFrom(
+ createLabel("key", "value"));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
@@ -251,6 +255,8 @@ TEST_F(OversubscriptionTest, FetchResourceUsage)
EXPECT_EQ(usage.get().executors(0).executor_info().executor_id(),
DEFAULT_EXECUTOR_ID);
ASSERT_EQ(usage.get().executors(0).statistics(), statistics);
+ ASSERT_EQ(task.executor().labels(),
+ usage.get().executors(0).executor_info().labels());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
@@ -669,6 +675,8 @@ TEST_F(OversubscriptionTest, QoSFetchResourceUsage)
EXPECT_NE(0u, offers.get().size());
TaskInfo task = createTask(offers.get()[0], "sleep 10", DEFAULT_EXECUTOR_ID);
+ task.mutable_executor()->mutable_labels()->add_labels()->CopyFrom(
+ createLabel("key", "value"));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
@@ -695,6 +703,8 @@ TEST_F(OversubscriptionTest, QoSFetchResourceUsage)
EXPECT_EQ(usage.get().executors(0).executor_info().executor_id(),
DEFAULT_EXECUTOR_ID);
ASSERT_EQ(usage.get().executors(0).statistics(), statistics);
+ ASSERT_EQ(task.executor().labels(),
+ usage.get().executors(0).executor_info().labels());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
http://git-wip-us.apache.org/repos/asf/mesos/blob/c7802c03/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 4a576b9..9e9dc35 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2788,6 +2788,100 @@ TEST_F(SlaveTest, DiscoveryInfoAndPorts)
}
+// This test verifies that executor labels are
+// exposed in the slave's state endpoint.
+TEST_F(SlaveTest, ExecutorLabels)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+ TestContainerizer containerizer(&exec);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(1);
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers->size());
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
+ task.mutable_resources()->MergeFrom(offers.get()[0].resources());
+ task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
+
+ // Add three labels to the executor (two of which share the same key).
+ Labels* labels = task.mutable_executor()->mutable_labels();
+
+ labels->add_labels()->CopyFrom(createLabel("key1", "value1"));
+ labels->add_labels()->CopyFrom(createLabel("key2", "value2"));
+ labels->add_labels()->CopyFrom(createLabel("key1", "value3"));
+
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .Times(1);
+
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY(status);
+ EXPECT_EQ(TASK_RUNNING, status->state());
+
+ // Verify label key and value in slave state endpoint.
+ Future<Response> response = process::http::get(
+ slave.get()->pid,
+ "state",
+ None(),
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response->body);
+ ASSERT_SOME(parse);
+
+ Result<JSON::Array> labels_ = parse->find<JSON::Array>(
+ "frameworks[0].executors[0].labels");
+ ASSERT_SOME(labels_);
+
+ // Verify the contents of labels.
+ EXPECT_EQ(3u, labels_->values.size());
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key1", "value1"))),
+ labels_->values[0]);
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key2", "value2"))),
+ labels_->values[1]);
+ EXPECT_EQ(JSON::Value(JSON::protobuf(createLabel("key1", "value3"))),
+ labels_->values[2]);
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+}
+
+
// This test verifies that label values can be set for tasks and that
// they are exposed over the slave state endpoint.
TEST_F(SlaveTest, TaskLabels)
[2/2] mesos git commit: Added a check in agent for invalid
oversubscribable resources.
Posted by bm...@apache.org.
Added a check in agent for invalid oversubscribable resources.
Review: https://reviews.apache.org/r/45941/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f63b4eea
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f63b4eea
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f63b4eea
Branch: refs/heads/master
Commit: f63b4eea4edcae7785f5290a936bfb75412c99b5
Parents: 2e76199
Author: Zhitao Li <zh...@gmail.com>
Authored: Tue Apr 12 11:50:47 2016 -0700
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Tue Apr 12 15:31:02 2016 -0700
----------------------------------------------------------------------
src/slave/slave.cpp | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f63b4eea/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index f3913c0..9de7e15 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -4972,6 +4972,12 @@ void Slave::_forwardOversubscribed(const Future<Resources>& oversubscribable)
VLOG(1) << "Received oversubscribable resources "
<< oversubscribable.get() << " from the resource estimator";
+ // Oversubscribable resources must be tagged as revocable.
+ //
+ // TODO(bmahler): Consider tagging input as revocable
+ // rather than rejecting and crashing here.
+ CHECK_EQ(oversubscribable.get(), oversubscribable->revocable());
+
// Calculate the latest allocation of oversubscribed resources.
// Note that this allocation value might be different from the
// master's view because new task/executor might be in flight from