You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2015/07/21 22:32:16 UTC
[1/4] mesos git commit: Added a helper testing functions for Labels.
Repository: mesos
Updated Branches:
refs/heads/master 65e1712bc -> b175cfcdd
Added a helper testing functions for Labels.
Added createLabel(key, value) return a new "key":"value" Label.
Review: https://reviews.apache.org/r/36574
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1afcf7c8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1afcf7c8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1afcf7c8
Branch: refs/heads/master
Commit: 1afcf7c897442b3f7dae7f220620a9df1a9c34b4
Parents: 65e1712
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Tue Jul 21 13:29:12 2015 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Tue Jul 21 13:29:15 2015 -0700
----------------------------------------------------------------------
src/tests/hook_tests.cpp | 14 +++-----
src/tests/master_tests.cpp | 79 +++++++++--------------------------------
src/tests/mesos.hpp | 10 ++++++
src/tests/slave_tests.cpp | 52 ++++++---------------------
4 files changed, 40 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/1afcf7c8/src/tests/hook_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hook_tests.cpp b/src/tests/hook_tests.cpp
index 09205fb..308ca9f 100644
--- a/src/tests/hook_tests.cpp
+++ b/src/tests/hook_tests.cpp
@@ -164,9 +164,8 @@ TEST_F(HookTest, VerifyMasterLaunchTaskHook)
// Add label which will be removed by the hook.
Labels* labels = task.mutable_labels();
- Label* label = labels->add_labels();
- label->set_key(testRemoveLabelKey);
- label->set_value(testRemoveLabelValue);
+ labels->add_labels()->CopyFrom(createLabel(
+ testRemoveLabelKey, testRemoveLabelValue));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -380,13 +379,8 @@ TEST_F(HookTest, VerifySlaveRunTaskHook)
// available by the end of the launch task sequence when hooks are
// used (to protect against hooks removing labels completely).
Labels* labels = task.mutable_labels();
- Label* label1 = labels->add_labels();
- label1->set_key("foo");
- label1->set_value("bar");
-
- Label* label2 = labels->add_labels();
- label2->set_key("bar");
- label2->set_value("baz");
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
vector<TaskInfo> tasks;
tasks.push_back(task);
http://git-wip-us.apache.org/repos/asf/mesos/blob/1afcf7c8/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 1e934c4..6bddc75 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -3061,17 +3061,9 @@ TEST_F(MasterTest, TaskLabels)
// Add three labels to the task (two of which share the same key).
Labels* labels = task.mutable_labels();
- Label* label1 = labels->add_labels();
- label1->set_key("foo");
- label1->set_value("bar");
-
- Label* label2 = labels->add_labels();
- label2->set_key("bar");
- label2->set_value("baz");
-
- Label* label3 = labels->add_labels();
- label3->set_key("bar");
- label3->set_value("qux");
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "qux"));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -3117,38 +3109,13 @@ TEST_F(MasterTest, TaskLabels)
JSON::Array labelsObject_ = labelsObject.get();
- // Verify the content of 'foo:bar' pair.
- Try<JSON::Value> expected = JSON::parse(
- "{"
- " \"key\":\"foo\","
- " \"value\":\"bar\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[0], expected.get());
-
-
- // Verify the content of 'bar:baz' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"bar\","
- " \"value\":\"baz\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[1], expected.get());
-
-
- // Verify the content of 'bar:qux' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"bar\","
- " \"value\":\"qux\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[2], expected.get());
-
+ // Verify the contents of 'foo:bar', 'bar:baz', and 'bar:qux' pairs.
+ EXPECT_EQ(labelsObject_.values[0],
+ JSON::Value(JSON::Protobuf(createLabel("foo", "bar"))));
+ EXPECT_EQ(labelsObject_.values[1],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "baz"))));
+ EXPECT_EQ(labelsObject_.values[2],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "qux"))));
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
@@ -3273,12 +3240,8 @@ TEST_F(MasterTest, TaskDiscoveryInfo)
// Add two labels to the discovery info.
Labels* labels = info->mutable_labels();
- Label* label1 = labels->add_labels();
- label1->set_key("clearance");
- label1->set_value("high");
- Label* label2 = labels->add_labels();
- label2->set_key("RPC");
- label2->set_value("yes");
+ labels->add_labels()->CopyFrom(createLabel("clearance", "high"));
+ labels->add_labels()->CopyFrom(createLabel("RPC", "yes"));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -3387,22 +3350,12 @@ TEST_F(MasterTest, TaskDiscoveryInfo)
EXPECT_EQ(2u, labelsArray_.values.size());
// Verify the content of 'clearance:high' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"clearance\","
- " \"value\":\"high\""
- "}");
- ASSERT_SOME(expected);
- EXPECT_EQ(expected.get(), labelsArray_.values[0]);
+ EXPECT_EQ(labelsArray_.values[0],
+ JSON::Value(JSON::Protobuf(createLabel("clearance", "high"))));
// Verify the content of 'RPC:yes' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"RPC\","
- " \"value\":\"yes\""
- "}");
- ASSERT_SOME(expected);
- EXPECT_EQ(expected.get(), labelsArray_.values[1]);
+ EXPECT_EQ(labelsArray_.values[1],
+ JSON::Value(JSON::Protobuf(createLabel("RPC", "yes"))));
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
http://git-wip-us.apache.org/repos/asf/mesos/blob/1afcf7c8/src/tests/mesos.hpp
----------------------------------------------------------------------
diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp
index 23d9841..f14b8f7 100644
--- a/src/tests/mesos.hpp
+++ b/src/tests/mesos.hpp
@@ -46,6 +46,7 @@
#include <stout/bytes.hpp>
#include <stout/foreach.hpp>
#include <stout/gtest.hpp>
+#include <stout/json.hpp>
#include <stout/lambda.hpp>
#include <stout/none.hpp>
#include <stout/option.hpp>
@@ -1443,6 +1444,15 @@ void ExpectNoFutureUnionProtobufs(
// StatusUpdate came from the corresponding task.
MATCHER_P(TaskStatusEq, task, "") { return arg.task_id() == task.task_id(); }
+
+inline Label createLabel(const std::string& key, const std::string& value)
+{
+ Label label;
+ label.set_key(key);
+ label.set_value(value);
+ return label;
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/1afcf7c8/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index a16e4f4..7266cf1 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2076,17 +2076,9 @@ TEST_F(SlaveTest, TaskLabels)
// Add three labels to the task (two of which share the same key).
Labels* labels = task.mutable_labels();
- Label* label1 = labels->add_labels();
- label1->set_key("foo");
- label1->set_value("bar");
-
- Label* label2 = labels->add_labels();
- label2->set_key("bar");
- label2->set_value("baz");
-
- Label* label3 = labels->add_labels();
- label3->set_key("bar");
- label3->set_value("qux");
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "qux"));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -2132,37 +2124,13 @@ TEST_F(SlaveTest, TaskLabels)
JSON::Array labelsObject_ = labelsObject.get();
- // Verify the content of 'foo:bar' pair.
- Try<JSON::Value> expected = JSON::parse(
- "{"
- " \"key\":\"foo\","
- " \"value\":\"bar\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[0], expected.get());
-
-
- // Verify the content of 'bar:baz' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"bar\","
- " \"value\":\"baz\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[1], expected.get());
-
-
- // Verify the content of 'bar:qux' pair.
- expected = JSON::parse(
- "{"
- " \"key\":\"bar\","
- " \"value\":\"qux\""
- "}");
-
- ASSERT_SOME(expected);
- EXPECT_EQ(labelsObject_.values[2], expected.get());
+ // Verify the contents of 'foo:bar', 'bar:baz', and 'bar:qux' pairs.
+ EXPECT_EQ(labelsObject_.values[0],
+ JSON::Value(JSON::Protobuf(createLabel("foo", "bar"))));
+ EXPECT_EQ(labelsObject_.values[1],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "baz"))));
+ EXPECT_EQ(labelsObject_.values[2],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "qux"))));
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
[2/4] mesos git commit: Added Labels to TaskStatus protobuf and
expose them via state.json.
Posted by be...@apache.org.
Added Labels to TaskStatus protobuf and expose them via state.json.
The labels would allow executors and Slave modules to pass in some
meta-data about the task to the framework and Mesos-DNS (via
state.json).
Review: https://reviews.apache.org/r/36575
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bbd3104b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bbd3104b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bbd3104b
Branch: refs/heads/master
Commit: bbd3104b0a42dca470716b6cedfcb9baa28be0fa
Parents: 1afcf7c
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Tue Jul 21 13:29:32 2015 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Tue Jul 21 13:29:34 2015 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 8 +++
src/common/http.cpp | 10 ++++
src/common/protobuf_utils.cpp | 7 ++-
src/common/protobuf_utils.hpp | 3 +-
src/tests/master_tests.cpp | 102 +++++++++++++++++++++++++++++++++++++
src/tests/slave_tests.cpp | 102 +++++++++++++++++++++++++++++++++++++
6 files changed, 230 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index cb24125..bcb38d9 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -965,6 +965,14 @@ message TaskStatus {
// (true) or unhealthy (false) according to the HealthCheck field in
// the command info.
optional bool healthy = 8;
+
+ // Labels are free-form key value pairs which are exposed through
+ // master and slave endpoints. Labels will not be interpreted or
+ // acted upon by Mesos itself. As opposed to the data field, labels
+ // will be kept in memory on master and slave processes. Therefore,
+ // labels should be used to tag TaskStatus message with light-weight
+ // meta-data.
+ optional Labels labels = 12;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/src/common/http.cpp
----------------------------------------------------------------------
diff --git a/src/common/http.cpp b/src/common/http.cpp
index 2bb1ba8..a74c51d 100644
--- a/src/common/http.cpp
+++ b/src/common/http.cpp
@@ -131,6 +131,16 @@ JSON::Object model(const TaskStatus& status)
object.values["state"] = TaskState_Name(status.state());
object.values["timestamp"] = status.timestamp();
+ if (status.has_labels()) {
+ JSON::Array array;
+ array.values.reserve(status.labels().labels().size()); // MESOS-2353.
+
+ foreach (const Label& label, status.labels().labels()) {
+ array.values.push_back(JSON::Protobuf(label));
+ }
+ object.values["labels"] = std::move(array);
+ }
+
return object;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/src/common/protobuf_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.cpp b/src/common/protobuf_utils.cpp
index 9ac81c3..9ba57a7 100644
--- a/src/common/protobuf_utils.cpp
+++ b/src/common/protobuf_utils.cpp
@@ -53,7 +53,8 @@ StatusUpdate createStatusUpdate(
const string& message = "",
const Option<TaskStatus::Reason>& reason = None(),
const Option<ExecutorID>& executorId = None(),
- const Option<bool>& healthy = None())
+ const Option<bool>& healthy = None(),
+ const Option<Labels>& labels = None())
{
StatusUpdate update;
@@ -102,6 +103,10 @@ StatusUpdate createStatusUpdate(
status->set_healthy(healthy.get());
}
+ if (labels.isSome()) {
+ status->mutable_labels()->CopyFrom(labels.get());
+ }
+
return update;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/src/common/protobuf_utils.hpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.hpp b/src/common/protobuf_utils.hpp
index afe5a85..64974c5 100644
--- a/src/common/protobuf_utils.hpp
+++ b/src/common/protobuf_utils.hpp
@@ -54,7 +54,8 @@ StatusUpdate createStatusUpdate(
const std::string& message = "",
const Option<TaskStatus::Reason>& reason = None(),
const Option<ExecutorID>& executorId = None(),
- const Option<bool>& healthy = None());
+ const Option<bool>& healthy = None(),
+ const Option<Labels>& labels = None());
Task createTask(
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 6bddc75..8a4e2c2 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -3127,6 +3127,108 @@ TEST_F(MasterTest, TaskLabels)
}
+// This test verifies that TaskStatus label values are exposed over
+// the master state endpoint.
+TEST_F(MasterTest, TaskStatusLabels)
+{
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ Try<PID<Slave>> slave = StartSlave(&exec);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(1);
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ TaskInfo task = createTask(offers.get()[0], "sleep 100", DEFAULT_EXECUTOR_ID);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ ExecutorDriver* execDriver;
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .WillOnce(SaveArg<0>(&execDriver));
+
+ Future<TaskInfo> execTask;
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(FutureArg<1>(&execTask));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY(execTask);
+
+ // Now send TASK_RUNNING update.
+ TaskStatus runningStatus;
+ runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id());
+ runningStatus.set_state(TASK_RUNNING);
+
+ // Add three labels to the task (two of which share the same key).
+ Labels* labels = runningStatus.mutable_labels();
+
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "qux"));
+
+ execDriver->sendStatusUpdate(runningStatus);
+
+ AWAIT_READY(status);
+
+ // Verify label key and value in master state.json.
+ Future<process::http::Response> response =
+ process::http::get(master.get(), "state.json");
+ AWAIT_READY(response);
+
+ EXPECT_SOME_EQ(
+ "application/json",
+ response.get().headers.get("Content-Type"));
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response.get().body);
+ ASSERT_SOME(parse);
+
+ Result<JSON::Array> labelsObject = parse.get().find<JSON::Array>(
+ "frameworks[0].tasks[0].statuses[0].labels");
+ EXPECT_SOME(labelsObject);
+
+ JSON::Array labelsObject_ = labelsObject.get();
+
+ // Verify the content of 'foo:bar' pair.
+ EXPECT_EQ(labelsObject_.values[0],
+ JSON::Value(JSON::Protobuf(createLabel("foo", "bar"))));
+ EXPECT_EQ(labelsObject_.values[1],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "baz"))));
+ EXPECT_EQ(labelsObject_.values[2],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "qux"))));
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
+}
+
+
// This tests the 'active' field in slave entries from state.json. We
// first verify an active slave, deactivate it and verify that the
// 'active' field is false.
http://git-wip-us.apache.org/repos/asf/mesos/blob/bbd3104b/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index 7266cf1..330a95b 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -2142,6 +2142,108 @@ TEST_F(SlaveTest, TaskLabels)
}
+// This test verifies that TaskStatus label values are exposed over
+// the slave state endpoint.
+TEST_F(SlaveTest, TaskStatusLabels)
+{
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ Try<PID<Slave>> slave = StartSlave(&exec);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(1);
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ TaskInfo task = createTask(offers.get()[0], "sleep 100", DEFAULT_EXECUTOR_ID);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ ExecutorDriver* execDriver;
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .WillOnce(SaveArg<0>(&execDriver));
+
+ Future<TaskInfo> execTask;
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(FutureArg<1>(&execTask));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY(execTask);
+
+ // Now send TASK_RUNNING update.
+ TaskStatus runningStatus;
+ runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id());
+ runningStatus.set_state(TASK_RUNNING);
+
+ // Add three labels to the task (two of which share the same key).
+ Labels* labels = runningStatus.mutable_labels();
+
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "qux"));
+
+ execDriver->sendStatusUpdate(runningStatus);
+
+ AWAIT_READY(status);
+
+ // Verify label key and value in master state.json.
+ Future<process::http::Response> response =
+ process::http::get(slave.get(), "state.json");
+ AWAIT_READY(response);
+
+ EXPECT_SOME_EQ(
+ "application/json",
+ response.get().headers.get("Content-Type"));
+
+ Try<JSON::Object> parse = JSON::parse<JSON::Object>(response.get().body);
+ ASSERT_SOME(parse);
+
+ Result<JSON::Array> labelsObject = parse.get().find<JSON::Array>(
+ "frameworks[0].executors[0].tasks[0].statuses[0].labels");
+ EXPECT_SOME(labelsObject);
+
+ JSON::Array labelsObject_ = labelsObject.get();
+
+ // Verify the contents of 'foo:bar', 'bar:baz', and 'bar:qux' pairs.
+ EXPECT_EQ(labelsObject_.values[0],
+ JSON::Value(JSON::Protobuf(createLabel("foo", "bar"))));
+ EXPECT_EQ(labelsObject_.values[1],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "baz"))));
+ EXPECT_EQ(labelsObject_.values[2],
+ JSON::Value(JSON::Protobuf(createLabel("bar", "qux"))));
+
+ EXPECT_CALL(exec, shutdown(_))
+ .Times(AtMost(1));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
+}
+
+
// Test that we can set the executors environment variables and it
// won't inhert the slaves.
TEST_F(SlaveTest, ExecutorEnvironmentVariables)
[4/4] mesos git commit: Exposed Docker container IP via labels.
Posted by be...@apache.org.
Exposed Docker container IP via labels.
This would allow mesos-dns to lookup container information such as its
IP address.
Review: https://reviews.apache.org/r/36585
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b175cfcd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b175cfcd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b175cfcd
Branch: refs/heads/master
Commit: b175cfcddec4aa4d66f1f85efc683046ae5ca723
Parents: a434ecc
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Tue Jul 21 13:30:21 2015 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Tue Jul 21 13:32:10 2015 -0700
----------------------------------------------------------------------
src/docker/docker.cpp | 14 +++++++++++++-
src/docker/docker.hpp | 14 ++++++++++++--
src/docker/executor.cpp | 5 +++++
src/tests/docker_containerizer_tests.cpp | 7 +++++++
4 files changed, 37 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b175cfcd/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 2793258..1367de8 100644
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -274,7 +274,19 @@ Try<Docker::Container> Docker::Container::create(const string& output)
bool started = startedAtValue.get().value != "0001-01-01T00:00:00Z";
- return Docker::Container(output, id, name, optionalPid, started);
+ Result<JSON::String> ipAddressValue =
+ json.find<JSON::String>("NetworkSettings.IPAddress");
+ if (ipAddressValue.isNone()) {
+ return Error("Unable to find NetworkSettings.IPAddress in container");
+ } else if (ipAddressValue.isError()) {
+ return Error(
+ "Error finding NetworkSettings.Name in container: " +
+ ipAddressValue.error());
+ }
+
+ string ipAddress = ipAddressValue.get().value;
+
+ return Docker::Container(output, id, name, optionalPid, started, ipAddress);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b175cfcd/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index fbae7bd..38e5299 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -69,14 +69,24 @@ public:
// needed since pid is empty when the container terminates.
const bool started;
+ // Returns the IPAddress of the container, or None if no IP has
+ // been not been assigned.
+ const Option<std::string> ipAddress;
+
private:
Container(
const std::string& output,
const std::string& id,
const std::string& name,
const Option<pid_t>& pid,
- bool started)
- : output(output), id(id), name(name), pid(pid), started(started) {}
+ bool started,
+ const Option<std::string>& ipAddress)
+ : output(output),
+ id(id),
+ name(name),
+ pid(pid),
+ started(started),
+ ipAddress(ipAddress) {}
};
class Image
http://git-wip-us.apache.org/repos/asf/mesos/blob/b175cfcd/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index cdcd8ee..256d53d 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -158,6 +158,11 @@ public:
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(TASK_RUNNING);
status.set_data(container.output);
+ if (container.ipAddress.isSome()) {
+ Label* label = status.mutable_labels()->add_labels();
+ label->set_key("Docker.NetworkSettings.IPAddress");
+ label->set_value(container.ipAddress.get());
+ }
driver->sendStatusUpdate(status);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b175cfcd/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index dc87d9f..5086af3 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -722,6 +722,13 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
Try<JSON::Array> parse = JSON::parse<JSON::Array>(statusRunning.get().data());
ASSERT_SOME(parse);
+ // Now verify that the Docker.NetworkSettings.IPAddress label is
+ // present.
+ ASSERT_TRUE(statusRunning.get().has_labels());
+ EXPECT_EQ(1, statusRunning.get().labels().labels().size());
+ EXPECT_EQ("Docker.NetworkSettings.IPAddress",
+ statusRunning.get().labels().labels(0).key());
+
ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
[3/4] mesos git commit: Added TaskStatus label decorator hook for
Slave.
Posted by be...@apache.org.
Added TaskStatus label decorator hook for Slave.
This allows Slave modules to expose some information to the frameworks
as well as Mesos-DNS via state.json.
Review: https://reviews.apache.org/r/36580
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a434ecc1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a434ecc1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a434ecc1
Branch: refs/heads/master
Commit: a434ecc14c9ebb4cb7dcb23c59306fff187ab9b5
Parents: bbd3104
Author: Kapil Arya <ka...@mesosphere.io>
Authored: Tue Jul 21 13:29:55 2015 -0700
Committer: Benjamin Hindman <be...@gmail.com>
Committed: Tue Jul 21 13:29:58 2015 -0700
----------------------------------------------------------------------
include/mesos/hook.hpp | 11 ++++
src/examples/test_hook_module.cpp | 25 ++++++++-
src/hook/manager.cpp | 31 +++++++++--
src/hook/manager.hpp | 4 ++
src/slave/slave.cpp | 7 +++
src/tests/hook_tests.cpp | 99 ++++++++++++++++++++++++++++++++++
6 files changed, 172 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/include/mesos/hook.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/hook.hpp b/include/mesos/hook.hpp
index 0995c24..bb5a635 100644
--- a/include/mesos/hook.hpp
+++ b/include/mesos/hook.hpp
@@ -79,6 +79,17 @@ public:
{
return Nothing();
}
+
+ // This hook is called from within slave when it receives a status
+ // update from the executor. A module implementing the hook creates
+ // and returns a set of labels. These labels overwrite the existing
+ // labels on the TaskStatus.
+ virtual Result<Labels> slaveTaskStatusLabelDecorator(
+ const FrameworkID& frameworkId,
+ const TaskStatus& status)
+ {
+ return None();
+ }
};
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/src/examples/test_hook_module.cpp
----------------------------------------------------------------------
diff --git a/src/examples/test_hook_module.cpp b/src/examples/test_hook_module.cpp
index d61cd55..c664b56 100644
--- a/src/examples/test_hook_module.cpp
+++ b/src/examples/test_hook_module.cpp
@@ -128,7 +128,6 @@ public:
return environment;
}
-
// This hook locates the file created by environment decorator hook
// and deletes it.
virtual Try<Nothing> slaveRemoveExecutorHook(
@@ -149,6 +148,30 @@ public:
return Nothing();
}
+
+
+ virtual Result<Labels> slaveTaskStatusLabelDecorator(
+ const FrameworkID& frameworkId,
+ const TaskStatus& status)
+ {
+ LOG(INFO) << "Executing 'slaveTaskStatusLabelDecorator' hook";
+
+ Labels labels;
+
+ // Set one known label.
+ Label* newLabel = labels.add_labels();
+ newLabel->set_key("bar");
+ newLabel->set_value("qux");
+
+ // Remove label which was set by test.
+ foreach (const Label& oldLabel, status.labels().labels()) {
+ if (oldLabel.key() != "foo") {
+ labels.add_labels()->CopyFrom(oldLabel);
+ }
+ }
+
+ return labels;
+ }
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/src/hook/manager.cpp
----------------------------------------------------------------------
diff --git a/src/hook/manager.cpp b/src/hook/manager.cpp
index 0108534..11e6b0a 100644
--- a/src/hook/manager.cpp
+++ b/src/hook/manager.cpp
@@ -111,7 +111,7 @@ Labels HookManager::masterLaunchTaskLabelDecorator(
TaskInfo taskInfo_ = taskInfo;
foreachpair (const string& name, Hook* hook, availableHooks) {
- const Result<Labels>& result =
+ const Result<Labels> result =
hook->masterLaunchTaskLabelDecorator(
taskInfo_,
frameworkInfo,
@@ -141,7 +141,7 @@ Labels HookManager::slaveRunTaskLabelDecorator(
TaskInfo taskInfo_ = taskInfo;
foreachpair (const string& name, Hook* hook, availableHooks) {
- const Result<Labels>& result =
+ const Result<Labels> result =
hook->slaveRunTaskLabelDecorator(taskInfo_, frameworkInfo, slaveInfo);
// NOTE: If the hook returns None(), the task labels won't be
@@ -164,7 +164,7 @@ Environment HookManager::slaveExecutorEnvironmentDecorator(
{
synchronized (mutex) {
foreachpair (const string& name, Hook* hook, availableHooks) {
- const Result<Environment>& result =
+ const Result<Environment> result =
hook->slaveExecutorEnvironmentDecorator(executorInfo);
// NOTE: If the hook returns None(), the environment won't be
@@ -188,7 +188,7 @@ void HookManager::slaveRemoveExecutorHook(
const ExecutorInfo& executorInfo)
{
foreachpair (const string& name, Hook* hook, availableHooks) {
- const Try<Nothing>& result =
+ const Try<Nothing> result =
hook->slaveRemoveExecutorHook(frameworkInfo, executorInfo);
if (result.isError()) {
LOG(WARNING) << "Slave remove executor hook failed for module '"
@@ -197,5 +197,28 @@ void HookManager::slaveRemoveExecutorHook(
}
}
+
+Labels HookManager::slaveTaskStatusLabelDecorator(
+ const FrameworkID& frameworkId,
+ TaskStatus status)
+{
+ synchronized (mutex) {
+ foreachpair (const string& name, Hook* hook, availableHooks) {
+ const Result<Labels> result =
+ hook->slaveTaskStatusLabelDecorator(frameworkId, status);
+
+ // NOTE: Labels remain unchanged if the hook returns None().
+ if (result.isSome()) {
+ status.mutable_labels()->CopyFrom(result.get());
+ } else if (result.isError()) {
+ LOG(WARNING) << "Slave TaskStatus label decorator hook failed for "
+ << "module '" << name << "': " << result.error();
+ }
+ }
+
+ return status.labels();
+ }
+}
+
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/src/hook/manager.hpp
----------------------------------------------------------------------
diff --git a/src/hook/manager.hpp b/src/hook/manager.hpp
index 47e8eb7..8153ce4 100644
--- a/src/hook/manager.hpp
+++ b/src/hook/manager.hpp
@@ -56,6 +56,10 @@ public:
static void slaveRemoveExecutorHook(
const FrameworkInfo& frameworkInfo,
const ExecutorInfo& executorInfo);
+
+ static Labels slaveTaskStatusLabelDecorator(
+ const FrameworkID& frameworkId,
+ TaskStatus status);
};
} // namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 2119b51..dc12c45 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -2708,6 +2708,13 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
return;
}
+ if (HookManager::hooksAvailable()) {
+ // Set TaskStatus labels from run task label decorator.
+ update.mutable_status()->mutable_labels()->CopyFrom(
+ HookManager::slaveTaskStatusLabelDecorator(
+ update.framework_id(), update.status()));
+ }
+
TaskStatus status = update.status();
Executor* executor = framework->getExecutor(status.task_id());
http://git-wip-us.apache.org/repos/asf/mesos/blob/a434ecc1/src/tests/hook_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hook_tests.cpp b/src/tests/hook_tests.cpp
index 308ca9f..6827dec 100644
--- a/src/tests/hook_tests.cpp
+++ b/src/tests/hook_tests.cpp
@@ -65,6 +65,7 @@ using std::vector;
using testing::_;
using testing::DoAll;
using testing::Return;
+using testing::SaveArg;
namespace mesos {
namespace internal {
@@ -422,6 +423,104 @@ TEST_F(HookTest, VerifySlaveRunTaskHook)
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
+
+// This test verifies that the slave task status label decorator can
+// add and remove labels from a TaskStatus during the status update
+// sequence. A TaskStatus with two labels ("foo":"bar" and
+// "bar":"baz") is sent from the executor. The labels get modified by
+// the slave hook to strip the "foo":"bar" pair and/ add a new
+// "baz":"qux" pair.
+TEST_F(HookTest, VerifySlaveTaskStatusLabelDecorator)
+{
+ Try<PID<Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockExecutor exec(DEFAULT_EXECUTOR_ID);
+
+ TestContainerizer containerizer(&exec);
+
+ Try<PID<Slave>> slave = StartSlave(&containerizer);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers.get().size());
+
+ // Start a task.
+ TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ ExecutorDriver* execDriver;
+ EXPECT_CALL(exec, registered(_, _, _, _))
+ .WillOnce(SaveArg<0>(&execDriver));
+
+ Future<TaskInfo> execTask;
+ EXPECT_CALL(exec, launchTask(_, _))
+ .WillOnce(FutureArg<1>(&execTask));
+
+ Future<TaskStatus> status;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status));
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY(execTask);
+
+ // Now send TASK_RUNNING update with two labels. The first label
+ // ("foo:bar") will be removed by the task status hook to ensure
+ // that it can remove labels. The second label will be preserved
+ // and forwarded to Master (and eventually to the framework).
+ // The hook also adds a new label with the same key but a different
+ // value ("bar:quz").
+ TaskStatus runningStatus;
+ runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id());
+ runningStatus.set_state(TASK_RUNNING);
+
+ // Add two labels to the TaskStatus
+ Labels* labels = runningStatus.mutable_labels();
+
+ labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
+ labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
+
+ execDriver->sendStatusUpdate(runningStatus);
+
+ AWAIT_READY(status);
+
+ // The master hook will hang an extra label off.
+ const Labels& labels_ = status.get().labels();
+
+ EXPECT_EQ(2, labels_.labels_size());
+
+ // The test hook will prepend a new "baz":"qux" label.
+ EXPECT_EQ(labels_.labels(0).key(), "bar");
+ EXPECT_EQ(labels_.labels(0).value(), "qux");
+
+ // And lastly, we only expect the "foo":"bar" pair to be stripped by
+ // the module. The last pair should be the original "bar":"baz"
+ // pair set by the test.
+ EXPECT_EQ(labels_.labels(1).key(), "bar");
+ EXPECT_EQ(labels_.labels(1).value(), "baz");
+
+ driver.stop();
+ driver.join();
+
+ Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {