You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by gr...@apache.org on 2017/08/19 06:29:39 UTC
[1/4] mesos git commit: Updated the docs for the V1 'HEARTBEAT' event.
Repository: mesos
Updated Branches:
refs/heads/master 4d0e692e7 -> 11ee081ee
Updated the docs for the V1 'HEARTBEAT' event.
Review: https://reviews.apache.org/r/61759/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a8d8d100
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a8d8d100
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a8d8d100
Branch: refs/heads/master
Commit: a8d8d1009bc634ad16021d5acc06b81ef711e69e
Parents: 4d0e692
Author: Quinn Leng <qu...@gmail.com>
Authored: Fri Aug 18 22:33:32 2017 -0700
Committer: Greg Mann <gr...@gmail.com>
Committed: Fri Aug 18 23:21:04 2017 -0700
----------------------------------------------------------------------
docs/operator-http-api.md | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a8d8d100/docs/operator-http-api.md
----------------------------------------------------------------------
diff --git a/docs/operator-http-api.md b/docs/operator-http-api.md
index c56f3b5..f6cfcf1 100644
--- a/docs/operator-http-api.md
+++ b/docs/operator-http-api.md
@@ -2366,6 +2366,19 @@ The following events are currently sent by the master. The canonical source of t
The first event sent by the master when a client sends a `SUBSCRIBE` request on the persistent connection. This includes a snapshot of the cluster state. See `SUBSCRIBE` above for details. Subsequent changes to the cluster state can result in more events (currently only `TASK_ADDED` and `TASK_UPDATED` are supported).
+### HEARTBEAT
+
+Periodically sent by the master to the subscriber according to 'Subscribed.heartbeat_interval_seconds'. If the subscriber does not receive any events (including heartbeats) for an extended period of time (e.g., 5 x heartbeat_interval_seconds), it is likely that the connection is lost or there is a network partition. In that case, the subscriber should close the existing subscription connection and resubscribe using a backoff strategy.
+
+```
+HEARTBEAT Event (JSON)
+
+<event-length>
+{
+ "type": "HEARTBEAT",
+}
+```
+
### TASK_ADDED
Sent whenever a task has been added to the master. This can happen either when a new task launch is processed by the master or when an agent re-registers with a failed over master.
[4/4] mesos git commit: Added the V1 `Teardown` call to teardown a
framework.
Posted by gr...@apache.org.
Added the V1 `Teardown` call to teardown a framework.
Review: https://reviews.apache.org/r/61222/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d855aecd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d855aecd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d855aecd
Branch: refs/heads/master
Commit: d855aecd324bc5b30eca995a0e443a96caa3c2ef
Parents: ef73c23
Author: Quinn Leng <qu...@gmail.com>
Authored: Fri Aug 18 22:34:39 2017 -0700
Committer: Greg Mann <gr...@gmail.com>
Committed: Fri Aug 18 23:21:09 2017 -0700
----------------------------------------------------------------------
include/mesos/master/master.proto | 9 +++++++++
include/mesos/v1/master/master.proto | 9 +++++++++
src/master/http.cpp | 29 ++++++++++++++++++++++++++---
src/master/master.hpp | 10 ++++++++++
src/master/validation.cpp | 6 ++++++
5 files changed, 60 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d855aecd/include/mesos/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/master/master.proto b/include/mesos/master/master.proto
index f41ae4b..b94e902 100644
--- a/include/mesos/master/master.proto
+++ b/include/mesos/master/master.proto
@@ -87,6 +87,8 @@ message Call {
GET_QUOTA = 28;
SET_QUOTA = 29; // See 'SetQuota' below.
REMOVE_QUOTA = 30; // See 'RemoveQuota' below.
+
+ TEARDOWN = 31; // See 'Teardown' below.
}
// Provides a snapshot of the current metrics tracked by the master.
@@ -188,6 +190,12 @@ message Call {
required string role = 1;
}
+ // Tears down a running framework by shutting down all tasks/executors and
+ // removing the framework.
+ message Teardown {
+ required FrameworkID framework_id = 1;
+ }
+
optional Type type = 1;
optional GetMetrics get_metrics = 2;
@@ -204,6 +212,7 @@ message Call {
optional StopMaintenance stop_maintenance = 13;
optional SetQuota set_quota = 14;
optional RemoveQuota remove_quota = 15;
+ optional Teardown teardown = 16;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/d855aecd/include/mesos/v1/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/master/master.proto b/include/mesos/v1/master/master.proto
index 0adc58e..7499fa4 100644
--- a/include/mesos/v1/master/master.proto
+++ b/include/mesos/v1/master/master.proto
@@ -87,6 +87,8 @@ message Call {
GET_QUOTA = 28;
SET_QUOTA = 29; // See 'SetQuota' below.
REMOVE_QUOTA = 30; // See 'RemoveQuota' below.
+
+ TEARDOWN = 31; // See 'Teardown' below.
}
// Provides a snapshot of the current metrics tracked by the master.
@@ -188,6 +190,12 @@ message Call {
required string role = 1;
}
+ // Tears down a running framework by shutting down all tasks/executors and
+ // removing the framework.
+ message Teardown {
+ required FrameworkID framework_id = 1;
+ }
+
optional Type type = 1;
optional GetMetrics get_metrics = 2;
@@ -204,6 +212,7 @@ message Call {
optional StopMaintenance stop_maintenance = 13;
optional SetQuota set_quota = 14;
optional RemoveQuota remove_quota = 15;
+ optional Teardown teardown = 16;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/d855aecd/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index e67abeb..ef4decc 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -776,6 +776,9 @@ Future<Response> Master::Http::api(
case mesos::master::Call::REMOVE_QUOTA:
return quotaHandler.remove(call, principal);
+
+ case mesos::master::Call::TEARDOWN:
+ return teardown(call, principal, acceptType);
}
UNREACHABLE();
@@ -3785,6 +3788,14 @@ Future<Response> Master::Http::teardown(
FrameworkID id;
id.set_value(value.get());
+ return _teardown(id, principal);
+}
+
+
+Future<Response> Master::Http::_teardown(
+ const FrameworkID& id,
+ const Option<Principal>& principal) const
+{
Framework* framework = master->getFramework(id);
if (framework == nullptr) {
@@ -3793,7 +3804,7 @@ Future<Response> Master::Http::teardown(
// Skip authorization if no ACLs were provided to the master.
if (master->authorizer.isNone()) {
- return _teardown(id);
+ return __teardown(id);
}
authorization::Request teardown;
@@ -3815,12 +3826,13 @@ Future<Response> Master::Http::teardown(
if (!authorized) {
return Forbidden();
}
- return _teardown(id);
+
+ return __teardown(id);
}));
}
-Future<Response> Master::Http::_teardown(const FrameworkID& id) const
+Future<Response> Master::Http::__teardown(const FrameworkID& id) const
{
Framework* framework = master->getFramework(id);
@@ -3835,6 +3847,17 @@ Future<Response> Master::Http::_teardown(const FrameworkID& id) const
}
+Future<Response> Master::Http::teardown(
+ const mesos::master::Call& call,
+ const Option<Principal>& principal,
+ ContentType contentType) const
+{
+ CHECK_EQ(mesos::master::Call::TEARDOWN, call.type());
+
+ return _teardown(call.teardown().framework_id(), principal);
+}
+
+
struct TaskComparator
{
static bool ascending(const Task* lhs, const Task* rhs)
http://git-wip-us.apache.org/repos/asf/mesos/blob/d855aecd/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index d9cfc42..d7c67d0 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1421,6 +1421,11 @@ private:
principal) const;
process::Future<process::http::Response> _teardown(
+ const FrameworkID& id,
+ const Option<process::http::authentication::Principal>&
+ principal) const;
+
+ process::Future<process::http::Response> __teardown(
const FrameworkID& id) const;
process::Future<process::http::Response> _updateMaintenanceSchedule(
@@ -1649,6 +1654,11 @@ private:
const Option<process::http::authentication::Principal>& principal,
ContentType contentType) const;
+ process::Future<process::http::Response> teardown(
+ const mesos::master::Call& call,
+ const Option<process::http::authentication::Principal>& principal,
+ ContentType contentType) const;
+
Master* master;
// NOTE: The quota specific pieces of the Operator API are factored
http://git-wip-us.apache.org/repos/asf/mesos/blob/d855aecd/src/master/validation.cpp
----------------------------------------------------------------------
diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index 279dd51..7c3247d 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -222,6 +222,12 @@ Option<Error> validate(
return Error("Expecting 'remove_quota' to be present");
}
return None();
+
+ case mesos::master::Call::TEARDOWN:
+ if (!call.has_teardown()) {
+ return Error("Expecting 'teardown' to be present");
+ }
+ return None();
}
UNREACHABLE();
[3/4] mesos git commit: Added a test for the `Teardown` call in V1
operator API.
Posted by gr...@apache.org.
Added a test for the `Teardown` call in V1 operator API.
Review: https://reviews.apache.org/r/61408/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/11ee081e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/11ee081e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/11ee081e
Branch: refs/heads/master
Commit: 11ee081ee578ea12e85799e00c5fe8b89eb6ea5f
Parents: d855aec
Author: Quinn Leng <qu...@gmail.com>
Authored: Fri Aug 18 22:34:43 2017 -0700
Committer: Greg Mann <gr...@gmail.com>
Committed: Fri Aug 18 23:21:09 2017 -0700
----------------------------------------------------------------------
src/tests/api_tests.cpp | 192 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 192 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/11ee081e/src/tests/api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index e5a6046..bc5024b 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -3002,6 +3002,198 @@ TEST_P(MasterAPITest, ReadFileInvalidPath)
}
+// This test verifies that when the operator API TEARDOWN call is made,
+// the framework is shutdown and removed. It also confirms that authorization
+// of this call is performed correctly.
+TEST_P(MasterAPITest, Teardown)
+{
+ ContentType contentType = GetParam();
+
+ ACLs acls;
+
+ // Only allow DEFAULT_CREDENTIAL to teardown frameworks.
+ {
+ mesos::ACL::TeardownFramework* acl = acls.add_teardown_frameworks();
+ acl->mutable_principals()->add_values(DEFAULT_CREDENTIAL_2.principal());
+ acl->mutable_framework_principals()->set_type(mesos::ACL::Entity::NONE);
+ }
+
+ {
+ mesos::ACL::TeardownFramework* acl = acls.add_teardown_frameworks();
+ acl->mutable_principals()->add_values(DEFAULT_CREDENTIAL.principal());
+ acl->mutable_framework_principals()->add_values(
+ DEFAULT_CREDENTIAL.principal());
+ }
+
+ master::Flags masterFlags = CreateMasterFlags();
+ Result<Authorizer*> authorizer = Authorizer::create(acls);
+
+ Try<Owned<cluster::Master>> master =
+ StartMaster(authorizer.get(), masterFlags);
+
+ auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
+ auto executor = std::make_shared<v1::MockHTTPExecutor>();
+
+ Future<RegisterSlaveMessage> registerSlaveMessage =
+ FUTURE_PROTOBUF(RegisterSlaveMessage(), _, _);
+
+ ExecutorID executorId = DEFAULT_EXECUTOR_ID;
+ TestContainerizer containerizer(executorId, executor);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
+ ASSERT_SOME(slave);
+
+ AWAIT_READY(registerSlaveMessage);
+
+ Future<Nothing> connected;
+ EXPECT_CALL(*scheduler, connected(_))
+ .WillOnce(FutureSatisfy(&connected))
+ .WillRepeatedly(Return()); // Ignore subsequent connections.
+
+ v1::scheduler::TestMesos mesos(
+ master.get()->pid,
+ contentType,
+ scheduler);
+
+ AWAIT_READY(connected);
+
+ Future<v1::scheduler::Event::Subscribed> subscribed;
+ EXPECT_CALL(*scheduler, subscribed(_, _))
+ .WillOnce(FutureArg<1>(&subscribed));
+
+ EXPECT_CALL(*scheduler, offers(_, _))
+ .WillRepeatedly(Return()); // Ignore offers.
+
+ EXPECT_CALL(*scheduler, heartbeat(_))
+ .WillRepeatedly(Return()); // Ignore heartbeats.
+
+ {
+ v1::scheduler::Call call;
+ call.set_type(v1::scheduler::Call::SUBSCRIBE);
+
+ v1::scheduler::Call::Subscribe* subscribe = call.mutable_subscribe();
+ v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_user("root");
+ subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
+
+ mesos.send(call);
+ }
+
+ AWAIT_READY(subscribed);
+
+ v1::FrameworkID frameworkId = subscribed->framework_id();
+
+ // There should be one framework in the response of the 'GET_FRAMEWORKS' call.
+ {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::GET_FRAMEWORKS);
+
+ Future<v1::master::Response> v1Response =
+ post(master.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::master::Response::GET_FRAMEWORKS, v1Response->type());
+
+ v1::master::Response::GetFrameworks frameworks =
+ v1Response->get_frameworks();
+
+ ASSERT_EQ(1, frameworks.frameworks_size());
+ }
+
+ // Send teardown with principal that is not authorized.
+ {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::TEARDOWN);
+
+ v1::master::Call::Teardown* teardown = v1Call.mutable_teardown();
+
+ teardown->mutable_framework_id()->CopyFrom(frameworkId);
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL_2),
+ serialize(contentType, v1Call),
+ stringify(contentType));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
+ }
+
+ // There should still be one framework in the response.
+ {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::GET_FRAMEWORKS);
+
+ Future<v1::master::Response> v1Response =
+ post(master.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::master::Response::GET_FRAMEWORKS, v1Response->type());
+
+ v1::master::Response::GetFrameworks frameworks =
+ v1Response->get_frameworks();
+
+ ASSERT_EQ(1, frameworks.frameworks_size());
+ }
+
+ Future<ShutdownFrameworkMessage> shutdownFrameworkMessage =
+ FUTURE_PROTOBUF(ShutdownFrameworkMessage(), _, _);
+
+ EXPECT_CALL(*scheduler, disconnected(_));
+
+ // Send the teardown call with the correct credential, it will teardown the
+ // framework.
+ {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::TEARDOWN);
+
+ v1::master::Call::Teardown* teardown = v1Call.mutable_teardown();
+
+ teardown->mutable_framework_id()->CopyFrom(frameworkId);
+
+ Future<http::Response> response = http::post(
+ master.get()->pid,
+ "api/v1",
+ createBasicAuthHeaders(DEFAULT_CREDENTIAL),
+ serialize(contentType, v1Call),
+ stringify(contentType));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
+ }
+
+ AWAIT_READY(shutdownFrameworkMessage);
+
+ // There should be one framework in the 'completed_frameworks' field of
+ // the response for the 'GET_FRAMEWORKS' call.
+ {
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::GET_FRAMEWORKS);
+
+ Future<v1::master::Response> v1Response =
+ post(master.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::master::Response::GET_FRAMEWORKS, v1Response->type());
+
+ v1::master::Response::GetFrameworks frameworks =
+ v1Response->get_frameworks();
+
+ ASSERT_EQ(0, frameworks.frameworks_size());
+ ASSERT_EQ(1, frameworks.completed_frameworks_size());
+ }
+
+ EXPECT_CALL(*executor, shutdown(_))
+ .Times(AtMost(1));
+
+ EXPECT_CALL(*executor, disconnected(_))
+ .Times(AtMost(1));
+}
+
+
class AgentAPITest
: public MesosTest,
public WithParamInterface<ContentType>
[2/4] mesos git commit: Added a heartbeat interval to the V1 master
operator API.
Posted by gr...@apache.org.
Added a heartbeat interval to the V1 master operator API.
Added the `heartbeat_interval_seconds` field to the V1
master operator API `Subscribed` event. Updated the heartbeat
test case to check the heartbeat interval.
Review: https://reviews.apache.org/r/61763/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ef73c23f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ef73c23f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ef73c23f
Branch: refs/heads/master
Commit: ef73c23f878eca9a5667a442284878e61fa91d5d
Parents: a8d8d10
Author: Quinn Leng <qu...@gmail.com>
Authored: Fri Aug 18 22:33:41 2017 -0700
Committer: Greg Mann <gr...@gmail.com>
Committed: Fri Aug 18 23:21:09 2017 -0700
----------------------------------------------------------------------
include/mesos/master/master.proto | 4 ++++
include/mesos/v1/master/master.proto | 4 ++++
src/master/http.cpp | 3 +++
src/tests/api_tests.cpp | 10 +++++++++-
4 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef73c23f/include/mesos/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/master/master.proto b/include/mesos/master/master.proto
index 7dc5881..f41ae4b 100644
--- a/include/mesos/master/master.proto
+++ b/include/mesos/master/master.proto
@@ -501,6 +501,10 @@ message Event {
// Snapshot of the entire cluster state. Further updates to the
// cluster state are sent as separate events on the stream.
optional Response.GetState get_state = 1;
+
+ // This value will be set if the master is sending heartbeats to
+ // subscribers. See the comment above on 'HEARTBEAT' for more details.
+ optional double heartbeat_interval_seconds = 2;
}
// Forwarded by the master when a task becomes known to it. This can happen
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef73c23f/include/mesos/v1/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/master/master.proto b/include/mesos/v1/master/master.proto
index db19c5c..0adc58e 100644
--- a/include/mesos/v1/master/master.proto
+++ b/include/mesos/v1/master/master.proto
@@ -499,6 +499,10 @@ message Event {
// Snapshot of the entire cluster state. Further updates to the
// cluster state are sent as separate events on the stream.
optional Response.GetState get_state = 1;
+
+ // This value will be set if the master is sending heartbeats to
+ // subscribers. See the comment above on 'HEARTBEAT' for more details.
+ optional double heartbeat_interval_seconds = 2;
}
// Forwarded by the master when a task becomes known to it. This can happen
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef73c23f/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index b09a9d0..e67abeb 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -853,6 +853,9 @@ Future<Response> Master::Http::subscribe(
executorsApprover,
rolesAcceptor));
+ event.mutable_subscribed()->set_heartbeat_interval_seconds(
+ DEFAULT_HEARTBEAT_INTERVAL.secs());
+
http.send<mesos::master::Event, v1::master::Event>(event);
mesos::master::Event heartbeatEvent;
http://git-wip-us.apache.org/repos/asf/mesos/blob/ef73c23f/src/tests/api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index 34480ea..e5a6046 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -2417,6 +2417,11 @@ TEST_P(MasterAPITest, Heartbeat)
AWAIT_READY(event);
EXPECT_EQ(v1::master::Event::SUBSCRIBED, event->get().type());
+ ASSERT_TRUE(event->get().subscribed().has_heartbeat_interval_seconds());
+ EXPECT_EQ(
+ DEFAULT_HEARTBEAT_INTERVAL.secs(),
+ event->get().subscribed().heartbeat_interval_seconds());
+
const v1::master::Response::GetState& getState =
event->get().subscribed().get_state();
@@ -2434,10 +2439,11 @@ TEST_P(MasterAPITest, Heartbeat)
event = decoder.read();
EXPECT_TRUE(event.isPending());
+ Clock::pause();
+
// Expects a heartbeat event after every heartbeat interval.
for (int i = 0; i < 10; i++) {
// Advance the clock to receive another heartbeat.
- Clock::pause();
Clock::advance(DEFAULT_HEARTBEAT_INTERVAL);
AWAIT_READY(event);
@@ -2446,6 +2452,8 @@ TEST_P(MasterAPITest, Heartbeat)
event = decoder.read();
EXPECT_TRUE(event.isPending());
}
+
+ Clock::resume();
}