You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/03/03 22:57:20 UTC
[1/3] mesos git commit: Added HTTP scheduler stream IDs.
Repository: mesos
Updated Branches:
refs/heads/master 09039136a -> e88815b52
Added HTTP scheduler stream IDs.
In some failure scenarios involving highly-available HTTP schedulers with
multiple instances, it's possible for a non-leading instance to successfully
make HTTP calls to the master. This patch enables the master to use HTTP
scheduler stream IDs to uniquely identify each HTTP subscription stream,
preventing any non-leading scheduler instance from making calls to the master.
The patch also adds stream ID support to the HTTP scheduler library.
Review: https://reviews.apache.org/r/44278/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4161e991
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4161e991
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4161e991
Branch: refs/heads/master
Commit: 4161e991a366072a874274269b96ea404b928707
Parents: 0903913
Author: Greg Mann <gr...@mesosphere.io>
Authored: Thu Mar 3 13:56:22 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Thu Mar 3 13:56:22 2016 -0800
----------------------------------------------------------------------
src/master/http.cpp | 31 ++++++++++++++++++++++++++++++-
src/master/master.hpp | 6 +++++-
src/scheduler/scheduler.cpp | 11 +++++++++++
3 files changed, 46 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4161e991/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 5e9e28e..8276baa 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -54,6 +54,7 @@
#include <stout/strings.hpp>
#include <stout/try.hpp>
#include <stout/utils.hpp>
+#include <stout/uuid.hpp>
#include "common/build.hpp"
#include "common/http.hpp"
@@ -422,6 +423,12 @@ Future<Response> Master::Http::scheduler(const Request& request) const
"'" + APPLICATION_PROTOBUF + "' or '" + APPLICATION_JSON + "'");
}
+ // Make sure that a stream ID was not included in the request headers.
+ if (request.headers.contains("Mesos-Stream-Id")) {
+ return BadRequest(
+ "Subscribe calls should not include the 'Mesos-Stream-Id' header");
+ }
+
Pipe pipe;
OK ok;
ok.headers["Content-Type"] = stringify(responseContentType);
@@ -429,7 +436,11 @@ Future<Response> Master::Http::scheduler(const Request& request) const
ok.type = Response::PIPE;
ok.reader = pipe.reader();
- HttpConnection http {pipe.writer(), responseContentType};
+ // Generate a stream ID and return it in the response.
+ UUID streamId = UUID::random();
+ ok.headers["Mesos-Stream-Id"] = streamId.toString();
+
+ HttpConnection http {pipe.writer(), responseContentType, streamId};
master->subscribe(http, call.subscribe());
return ok;
@@ -447,6 +458,24 @@ Future<Response> Master::Http::scheduler(const Request& request) const
return Forbidden("Framework is not subscribed");
}
+ if (framework->http.isNone()) {
+ return Forbidden("Framework is not connected via HTTP");
+ }
+
+ // This isn't a `SUBSCRIBE` call, so the request should include a stream ID.
+ if (!request.headers.contains("Mesos-Stream-Id")) {
+ return BadRequest(
+ "All non-subscribe calls should include the 'Mesos-Stream-Id' header");
+ }
+
+ if (request.headers.at("Mesos-Stream-Id") !=
+ framework->http.get().streamId.toString()) {
+ return BadRequest(
+ "The stream ID included in this request didn't match the stream ID "
+ "currently associated with framework ID '"
+ + framework->id().value() + "'");
+ }
+
switch (call.type()) {
case scheduler::Call::TEARDOWN:
master->removeFramework(framework);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4161e991/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 52b2cea..ea26670 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -56,6 +56,7 @@
#include <stout/multihashmap.hpp>
#include <stout/option.hpp>
#include <stout/recordio.hpp>
+#include <stout/uuid.hpp>
#include "common/http.hpp"
#include "common/protobuf_utils.hpp"
@@ -1600,9 +1601,11 @@ inline std::ostream& operator<<(
struct HttpConnection
{
HttpConnection(const process::http::Pipe::Writer& _writer,
- ContentType _contentType)
+ ContentType _contentType,
+ UUID _streamId)
: writer(_writer),
contentType(_contentType),
+ streamId(_streamId),
encoder(lambda::bind(serialize, contentType, lambda::_1)) {}
// Converts the message to an Event before sending.
@@ -1626,6 +1629,7 @@ struct HttpConnection
process::http::Pipe::Writer writer;
ContentType contentType;
+ UUID streamId;
::recordio::Encoder<v1::scheduler::Event> encoder;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/4161e991/src/scheduler/scheduler.cpp
----------------------------------------------------------------------
diff --git a/src/scheduler/scheduler.cpp b/src/scheduler/scheduler.cpp
index 7ea1c25..2c6bf0d 100644
--- a/src/scheduler/scheduler.cpp
+++ b/src/scheduler/scheduler.cpp
@@ -250,6 +250,11 @@ public:
// Send a streaming request for Subscribe call.
response = connections->subscribe.send(request, true);
} else {
+ CHECK_SOME(streamId);
+
+ // Set the stream ID associated with this connection.
+ request.headers["Mesos-Stream-Id"] = streamId.get().toString();
+
response = connections->nonSubscribe.send(request);
}
@@ -506,6 +511,11 @@ protected:
subscribed = SubscribedResponse {reader, decoder};
+ // Responses to SUBSCRIBE calls should always include a stream ID.
+ CHECK(response.get().headers.contains("Mesos-Stream-Id"));
+
+ streamId = UUID::fromString(response.get().headers.at("Mesos-Stream-Id"));
+
read();
return;
@@ -666,6 +676,7 @@ private:
shared_ptr<MasterDetector> detector;
queue<Event> events;
Option<::URL> master;
+ Option<UUID> streamId;
// Master detection future.
process::Future<Option<mesos::MasterInfo>> detection;
[2/3] mesos git commit: Added tests involving HTTP scheduler stream
IDs.
Posted by vi...@apache.org.
Added tests involving HTTP scheduler stream IDs.
Three new tests have been added in this patch:
SchedulerHttpApiTest.TeardownWithoutStreamId,
SchedulerHttpApiTest.TeardownWrongStreamId, and
SchedulerHttpApiTest.SubscribeWithStreamId.
Review: https://reviews.apache.org/r/44290/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7c50483f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7c50483f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7c50483f
Branch: refs/heads/master
Commit: 7c50483fe127409fcf4b4a8a17ac74855841819b
Parents: 4161e99
Author: Greg Mann <gr...@mesosphere.io>
Authored: Thu Mar 3 13:56:31 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Thu Mar 3 13:56:31 2016 -0800
----------------------------------------------------------------------
src/tests/scheduler_http_api_tests.cpp | 240 ++++++++++++++++++++++++++++
1 file changed, 240 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7c50483f/src/tests/scheduler_http_api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/scheduler_http_api_tests.cpp b/src/tests/scheduler_http_api_tests.cpp
index a8d660a..dfb0f51 100644
--- a/src/tests/scheduler_http_api_tests.cpp
+++ b/src/tests/scheduler_http_api_tests.cpp
@@ -29,6 +29,7 @@
#include <stout/json.hpp>
#include <stout/lambda.hpp>
#include <stout/recordio.hpp>
+#include <stout/uuid.hpp>
#include "common/http.hpp"
#include "common/recordio.hpp"
@@ -286,6 +287,8 @@ TEST_P(SchedulerHttpApiTest, Subscribe)
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
+ ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
+ EXPECT_NE("", response.get().headers.at("Mesos-Stream-Id"));
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
@@ -324,6 +327,42 @@ TEST_P(SchedulerHttpApiTest, Subscribe)
}
+// This test verifies that the client will receive a `BadRequest` response if it
+// includes a stream ID header with a subscribe call.
+TEST_P(SchedulerHttpApiTest, SubscribeWithStreamId)
+{
+ // HTTP schedulers cannot yet authenticate.
+ master::Flags flags = CreateMasterFlags();
+ flags.authenticate_frameworks = false;
+
+ Try<PID<Master>> master = StartMaster(flags);
+ ASSERT_SOME(master);
+
+ Call call;
+ call.set_type(Call::SUBSCRIBE);
+
+ Call::Subscribe* subscribe = call.mutable_subscribe();
+ subscribe->mutable_framework_info()->CopyFrom(DEFAULT_V1_FRAMEWORK_INFO);
+
+ // Retrieve the parameter passed as content type to this test.
+ const string contentType = GetParam();
+ process::http::Headers headers;
+ headers["Accept"] = contentType;
+ headers["Mesos-Stream-Id"] = UUID::random().toString();
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+
+ Shutdown();
+}
+
+
// This test verifies if the scheduler can subscribe on retrying,
// e.g. after a ZK blip.
TEST_P(SchedulerHttpApiTest, SubscribedOnRetry)
@@ -705,6 +744,207 @@ TEST_F(SchedulerHttpApiTest, GetRequest)
AWAIT_EXPECT_RESPONSE_STATUS_EQ(MethodNotAllowed({"POST"}).status, response);
}
+
+// This test verifies that the scheduler will receive a `BadRequest` response
+// when a teardown call is made without including a stream ID header.
+TEST_P(SchedulerHttpApiTest, TeardownWithoutStreamId)
+{
+ // HTTP schedulers cannot yet authenticate.
+ master::Flags flags = CreateMasterFlags();
+ flags.authenticate_frameworks = false;
+
+ Try<PID<Master>> master = StartMaster(flags);
+ ASSERT_SOME(master);
+
+ // Retrieve the parameter passed as content type to this test.
+ const string contentType = GetParam();
+ process::http::Headers headers;
+ headers["Accept"] = contentType;
+
+ v1::FrameworkID frameworkId;
+
+ {
+ Call call;
+ call.set_type(Call::SUBSCRIBE);
+
+ Call::Subscribe* subscribe = call.mutable_subscribe();
+ subscribe->mutable_framework_info()->CopyFrom(DEFAULT_V1_FRAMEWORK_INFO);
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
+ ASSERT_EQ(Response::PIPE, response.get().type);
+ ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
+
+ Option<Pipe::Reader> reader = response.get().reader;
+ ASSERT_SOME(reader);
+
+ auto deserializer = lambda::bind(
+ &SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
+
+ Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
+
+ Future<Result<Event>> event = responseDecoder.read();
+ AWAIT_READY(event);
+ ASSERT_SOME(event.get());
+
+ // Check event type is subscribed and the framework ID is set.
+ ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
+ EXPECT_NE("", event.get().get().subscribed().framework_id().value());
+
+ frameworkId = event.get().get().subscribed().framework_id();
+ }
+
+ {
+ // Send a TEARDOWN call without a stream ID.
+ Call call;
+ call.set_type(Call::TEARDOWN);
+ call.mutable_framework_id()->CopyFrom(frameworkId);
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+ }
+
+ Shutdown();
+}
+
+
+// This test verifies that the scheduler will receive a `BadRequest` response
+// when a teardown call is made with an incorrect stream ID header.
+TEST_P(SchedulerHttpApiTest, TeardownWrongStreamId)
+{
+ // HTTP schedulers cannot yet authenticate.
+ master::Flags flags = CreateMasterFlags();
+ flags.authenticate_frameworks = false;
+
+ Try<PID<Master>> master = StartMaster(flags);
+ ASSERT_SOME(master);
+
+ // Retrieve the parameter passed as content type to this test.
+ const string contentType = GetParam();
+ process::http::Headers headers;
+ headers["Accept"] = contentType;
+
+ v1::FrameworkID frameworkId;
+ string streamId;
+
+ // Subscribe once to get a valid stream ID.
+ {
+ Call call;
+ call.set_type(Call::SUBSCRIBE);
+
+ Call::Subscribe* subscribe = call.mutable_subscribe();
+ subscribe->mutable_framework_info()->CopyFrom(DEFAULT_V1_FRAMEWORK_INFO);
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
+ ASSERT_EQ(Response::PIPE, response.get().type);
+ ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
+
+ streamId = response.get().headers.at("Mesos-Stream-Id");
+
+ Option<Pipe::Reader> reader = response.get().reader;
+ ASSERT_SOME(reader);
+
+ auto deserializer = lambda::bind(
+ &SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
+
+ Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
+
+ Future<Result<Event>> event = responseDecoder.read();
+ AWAIT_READY(event);
+ ASSERT_SOME(event.get());
+
+ // Check that the event type is subscribed and the framework ID is set.
+ ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
+ EXPECT_NE("", event.get().get().subscribed().framework_id().value());
+
+ frameworkId = event.get().get().subscribed().framework_id();
+ }
+
+ // Subscribe again to invalidate the first stream ID and acquire another one.
+ {
+ Call call;
+ call.set_type(Call::SUBSCRIBE);
+
+ Call::Subscribe* subscribe = call.mutable_subscribe();
+ subscribe->mutable_framework_info()->CopyFrom(DEFAULT_V1_FRAMEWORK_INFO);
+
+ // Set the framework ID in the subscribe call.
+ call.mutable_framework_id()->CopyFrom(frameworkId);
+ subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+ AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
+ ASSERT_EQ(Response::PIPE, response.get().type);
+ ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
+
+ // Make sure that the new stream ID is different.
+ ASSERT_NE(streamId, response.get().headers.at("Mesos-Stream-Id"));
+
+ Option<Pipe::Reader> reader = response.get().reader;
+ ASSERT_SOME(reader);
+
+ auto deserializer = lambda::bind(
+ &SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
+
+ Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
+
+ Future<Result<Event>> event = responseDecoder.read();
+ AWAIT_READY(event);
+ ASSERT_SOME(event.get());
+
+ ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
+ EXPECT_NE("", event.get().get().subscribed().framework_id().value());
+ }
+
+ {
+ Call call;
+ call.set_type(Call::TEARDOWN);
+ call.mutable_framework_id()->CopyFrom(frameworkId);
+
+ // Send the first (now incorrect) stream ID with the teardown call.
+ headers["Mesos-Stream-Id"] = streamId;
+
+ Future<Response> response = process::http::streaming::post(
+ master.get(),
+ "api/v1/scheduler",
+ headers,
+ serialize(call, contentType),
+ contentType);
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
+ }
+
+ Shutdown();
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[3/3] mesos git commit: Added stream IDs to the HTTP API docs.
Posted by vi...@apache.org.
Added stream IDs to the HTTP API docs.
Review: https://reviews.apache.org/r/44279/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e88815b5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e88815b5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e88815b5
Branch: refs/heads/master
Commit: e88815b5201d152fe68d6e70671a681f56839bd0
Parents: 7c50483
Author: Greg Mann <gr...@mesosphere.io>
Authored: Thu Mar 3 13:56:38 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Thu Mar 3 13:56:38 2016 -0800
----------------------------------------------------------------------
CHANGELOG | 1 +
docs/scheduler-http-api.md | 19 +++++++++++++++++--
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/e88815b5/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 67bfbb6..eb708cb 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -37,6 +37,7 @@ This release contains several new features:
* [MESOS-4547] - Introduce TASK_KILLING state.
* [MESOS-4712] - Remove 'force' field from the Subscribe Call in v1 Scheduler API.
* [MESOS-4591] - Change the object of ReserveResources and CreateVolume ACLs to `roles`.
+ * [MESOS-3583] - Add stream IDs for HTTP schedulers.
** Deprecations:
* All Mesos HTTP endpoints with ".json" suffix (e.g., /master/state.json) are
http://git-wip-us.apache.org/repos/asf/mesos/blob/e88815b5/docs/scheduler-http-api.md
----------------------------------------------------------------------
diff --git a/docs/scheduler-http-api.md b/docs/scheduler-http-api.md
index c144b65..6926c27 100644
--- a/docs/scheduler-http-api.md
+++ b/docs/scheduler-http-api.md
@@ -21,7 +21,7 @@ All the subsequent (non subscribe) requests to "/scheduler" endpoint (see detail
## Calls
-The following calls are currently accepted by the master. The canonical source of this information is [scheduler.proto](https://github.com/apache/mesos/blob/master/include/mesos/v1/scheduler/scheduler.proto) (NOTE: The protobuf definitions are subject to change before the beta API is finalized). Note that when sending JSON encoded Calls, schedulers should encode raw bytes in Base64 and strings in UTF-8.
+The following calls are currently accepted by the master. The canonical source of this information is [scheduler.proto](https://github.com/apache/mesos/blob/master/include/mesos/v1/scheduler/scheduler.proto) (NOTE: The protobuf definitions are subject to change before the beta API is finalized). Note that when sending JSON encoded Calls, schedulers should encode raw bytes in Base64 and strings in UTF-8. All non-`SUBSCRIBE` calls should include the `Mesos-Stream-Id` header, explained in the [`SUBSCRIBE`](#subscribe) section. `SUBSCRIBE` calls should never include the `Mesos-Stream-Id` header.
<a id="recordio-response-format"></a>
@@ -73,7 +73,7 @@ Network intermediaries e.g. proxies are free to change the chunk boundaries and
This is the first step in the communication process between the scheduler and the master. This is also to be considered as subscription to the "/scheduler" events stream.
-To subscribe with the master, the scheduler sends a HTTP POST request with encoded `SUBSCRIBE` message with the required FrameworkInfo. Note that if "subscribe.framework_info.id" is not set, master considers the scheduler as a new one and subscribes it by assigning it a FrameworkID. The HTTP response is a stream with RecordIO encoding, with the first event being `SUBSCRIBED` event (see details in **Events** section).
+To subscribe with the master, the scheduler sends a HTTP POST request with encoded `SUBSCRIBE` message with the required FrameworkInfo. Note that if "subscribe.framework_info.id" is not set, master considers the scheduler as a new one and subscribes it by assigning it a FrameworkID. The HTTP response is a stream with RecordIO encoding, with the first event being `SUBSCRIBED` event (see details in **Events** section). The response also includes the `Mesos-Stream-Id` header, which is used by the master to uniquely identify the subscribed scheduler instance. This stream ID header should be included in all subsequent non-`SUBSCRIBE` calls sent over this subscription connection to the master. The value of `Mesos-Stream-Id` is guaranteed to be equal to or less than 128 bytes in length.
```
SUBSCRIBE Request (JSON):
@@ -101,6 +101,7 @@ HTTP/1.1 200 OK
Content-Type: application/json
Transfer-Encoding: chunked
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
<event length>
{
@@ -122,6 +123,8 @@ If subscription fails for whatever reason (e.g., invalid request), a HTTP 4xx re
Scheduler must make additional HTTP requests to the "/scheduler" endpoint only after it has opened a persistent connection to it by sending a `SUBSCRIBE` request and received a `SUBSCRIBED` response. Calls made without subscription will result in a "403 Forbidden" instead of a "202 Accepted" response. A scheduler might also receive a "400 Bad Request" response if the HTTP request is malformed (e.g., malformed HTTP headers).
+Note that the `Mesos-Stream-Id` header should **never** be included with a `SUBSCRIBE` call; the master will always provide a new unique stream ID for each subscription.
+
### TEARDOWN
Sent by the scheduler when it wants to tear itself down. When Mesos receives this request it will shut down all executors (and consequently kill tasks) and remove persistent volumes (if requested). It then removes the framework and closes all open connections from this scheduler to the Master.
@@ -131,6 +134,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -150,6 +154,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -178,6 +183,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -205,6 +211,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -225,6 +232,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -249,6 +257,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -273,6 +282,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -298,6 +308,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -325,6 +336,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -350,6 +362,7 @@ POST /api/v1/scheduler HTTP/1.1
Host: masterhost:5050
Content-Type: application/json
+Mesos-Stream-Id: 130ae4e3-6b13-4ef4-baa9-9f2e85c3e9af
{
"framework_id" : {"value" : "12220-3440-12532-2345"},
@@ -510,6 +523,8 @@ If the scheduler realizes that its subscription connection to "/scheduler" is br
If the master does not realize that the subscription connection is broken, but the scheduler realizes it, the scheduler might open a new persistent connection to
"/scheduler" via `SUBSCRIBE`. In this case, the master closes the existing subscription connection and allows subscription on the new connection. The invariant here is that only one persistent subscription connection for a given framework ID is allowed on the master.
+The master uses the `Mesos-Stream-Id` header to distinguish scheduler instances from one another. In the case of highly-available schedulers with multiple instances, this can prevent unwanted behavior in certain failure scenarios. Each unique `Mesos-Stream-Id` is valid only for the life of a single subscription connection. Each response to a `SUBSCRIBE` request contains a `Mesos-Stream-Id`, and this ID must be included with all subsequent non-subscribe calls sent over that subscription connection. Whenever a new subscription connection is established, a new stream ID is generated and should be used for the life of that connection.
+
### Network partitions
In the case of a network partition, the subscription connection between the scheduler and master might not necessarily break. To be able to detect this scenario, master periodically (e.g., 15s) sends `HEARTBEAT` events (similar to Twitter's Streaming API). If a scheduler doesn't receive a bunch (e.g., 5) of these heartbeats within a time window, it should immediately disconnect and try to resubscribe. It is highly recommended for schedulers to use an exponential backoff strategy (e.g., up to a maximum of 15s) to avoid overwhelming the master while reconnecting. Schedulers can use a similar timeout (e.g., 75s) for receiving responses to any HTTP requests.