[1/3] mesos git commit: Introduced a `reconnect` method on the scheduler library.

[vi...@apache.org]

Repository: mesos
Updated Branches:
  refs/heads/master 0a4e1e467 -> c6d618222


Introduced a `reconnect` method on the scheduler library.

This change adds a `reconnect` method that allows schedulers
to force a reconnection with the master. This is useful when
the scheduler detects that there is a one-way partition with
the master (e.g., lack of `HEARTBEAT` events) and wants to
trigger a reconnection rather than relying on the `disconnected`
callback.

Review: https://reviews.apache.org/r/45408/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/140c82a4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/140c82a4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/140c82a4

Branch: refs/heads/master
Commit: 140c82a41b1353fd838f19f2eaa5bb8198d704fb
Parents: 0a4e1e4
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Wed Mar 30 12:00:27 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Wed Mar 30 12:00:27 2016 -0700

----------------------------------------------------------------------
 include/mesos/v1/scheduler.hpp | 13 +++++++++++++
 src/scheduler/scheduler.cpp    | 23 +++++++++++++++++++++++
 2 files changed, 36 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/140c82a4/include/mesos/v1/scheduler.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/v1/scheduler.hpp b/include/mesos/v1/scheduler.hpp
index 765935e..bfc54c7 100644
--- a/include/mesos/v1/scheduler.hpp
+++ b/include/mesos/v1/scheduler.hpp
@@ -73,6 +73,19 @@ public:
   // disconnected).
   virtual void send(const Call& call);
 
+  // Force a reconnection with the master.
+  //
+  // In the case of a one-way network partition, the connection between the
+  // scheduler and master might not necessarily break. If the scheduler detects
+  // a partition, due to lack of `HEARTBEAT` events (e.g., 5) within a time
+  // window, it can explicitly ask the library to force a reconnection with
+  // the master.
+  //
+  // This call would be ignored if the scheduler is already disconnected with
+  // the master (e.g., no new master has been elected). Otherwise, the scheduler
+  // would get a 'disconnected' callback followed by a 'connected' callback.
+  virtual void reconnect();
+
 protected:
   // NOTE: This constructor is used for testing.
   Mesos(

http://git-wip-us.apache.org/repos/asf/mesos/blob/140c82a4/src/scheduler/scheduler.cpp
----------------------------------------------------------------------
diff --git a/src/scheduler/scheduler.cpp b/src/scheduler/scheduler.cpp
index 56476c3..5263a9c 100644
--- a/src/scheduler/scheduler.cpp
+++ b/src/scheduler/scheduler.cpp
@@ -267,6 +267,23 @@ public:
                          lambda::_1));
   }
 
+  void reconnect()
+  {
+    // Ignore the reconnection request if we are currently disconnected
+    // from the master.
+    if (state == DISCONNECTED) {
+      VLOG(1) << "Ignoring reconnect request from scheduler since we are"
+              << " disconnected";
+
+      return;
+    }
+
+    CHECK_SOME(connectionId);
+
+    disconnected(connectionId.get(),
+                 "Received reconnect request from scheduler");
+  }
+
 protected:
   virtual void initialize()
   {
@@ -754,6 +771,12 @@ void Mesos::send(const Call& call)
 }
 
 
+void Mesos::reconnect()
+{
+  dispatch(process, &MesosProcess::reconnect);
+}
+
+
 void Mesos::stop()
 {
   if (process != NULL) {

[3/3] mesos git commit: Documented when to invoke `send` using the scheduler library.

[vi...@apache.org]

Documented when to invoke `send` using the scheduler library.

Trivial change to add a comment to invoke `send` only after
receiving the `connected` callback. If not, all calls would
be dropped while disconnected.

Review: https://reviews.apache.org/r/45410/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c6d61822
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c6d61822
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c6d61822

Branch: refs/heads/master
Commit: c6d6182223816607038b76f57ece2e1453f82d8b
Parents: bae4f13
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Wed Mar 30 12:00:35 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Wed Mar 30 12:00:35 2016 -0700

----------------------------------------------------------------------
 include/mesos/v1/scheduler.hpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/c6d61822/include/mesos/v1/scheduler.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/v1/scheduler.hpp b/include/mesos/v1/scheduler.hpp
index bfc54c7..5264732 100644
--- a/include/mesos/v1/scheduler.hpp
+++ b/include/mesos/v1/scheduler.hpp
@@ -47,7 +47,9 @@ class MesosProcess; // Forward declaration.
 // 'received' which will get invoked _serially_ when it's determined
 // that we've connected (i.e., detected master), disconnected
 // (i.e, detected no master), or received events from the master.
-// Note that we drop events while disconnected.
+// The library reconnects with the master upon a disconnection.
+//
+// NOTE: All calls and events are dropped while disconnected.
 class Mesos
 {
 public:
@@ -67,6 +69,10 @@ public:
 
   // Attempts to send a call to the master.
   //
+  // The scheduler should only invoke this method once it has received the
+  // 'connected' callback. Otherwise, all calls would be dropped while
+  // disconnected.
+  //
   // Some local validation of calls is performed which may generate
   // events without ever being sent to the master. This includes when
   // calls are sent but no master is currently detected (i.e., we're

[2/3] mesos git commit: Added test for `reconnect` functionality.

[vi...@apache.org]

Added test for `reconnect` functionality.

This change adds a trivial test for testing the `reconnect`
method on the scheduler library.

Review: https://reviews.apache.org/r/45409/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/bae4f13f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/bae4f13f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/bae4f13f

Branch: refs/heads/master
Commit: bae4f13f7ad9f35202866b398b7bdd278f6d3fea
Parents: 140c82a
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Wed Mar 30 12:00:31 2016 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Wed Mar 30 12:00:31 2016 -0700

----------------------------------------------------------------------
 src/tests/scheduler_tests.cpp | 64 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/bae4f13f/src/tests/scheduler_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/scheduler_tests.cpp b/src/tests/scheduler_tests.cpp
index 917058f..0c66f65 100644
--- a/src/tests/scheduler_tests.cpp
+++ b/src/tests/scheduler_tests.cpp
@@ -1478,6 +1478,70 @@ TEST_P(SchedulerTest, Request)
 }
 
 
+// This test verifies that the scheduler is able to force a reconnection with
+// the master.
+TEST_P(SchedulerTest, SchedulerReconnect)
+{
+  master::Flags flags = CreateMasterFlags();
+  flags.authenticate_frameworks = false;
+
+  Try<Owned<cluster::Master>> master = StartMaster(flags);
+  ASSERT_SOME(master);
+
+  auto scheduler = std::make_shared<MockV1HTTPScheduler>();
+  auto detector = std::make_shared<StandaloneMasterDetector>(master.get()->pid);
+
+  Future<Nothing> connected;
+  EXPECT_CALL(*scheduler, connected(_))
+    .WillOnce(FutureSatisfy(&connected));
+
+  ContentType contentType = GetParam();
+
+  scheduler::TestV1Mesos mesos(
+      master.get()->pid, contentType, scheduler, detector);
+
+  AWAIT_READY(connected);
+
+  Future<Nothing> disconnected;
+  EXPECT_CALL(*scheduler, disconnected(_))
+    .WillOnce(FutureSatisfy(&disconnected));
+
+  EXPECT_CALL(*scheduler, connected(_))
+    .WillOnce(FutureSatisfy(&connected));
+
+  // Force a reconnection with the master. This should result in a
+  // `disconnected` callback followed by a `connected` callback.
+  mesos.reconnect();
+
+  AWAIT_READY(disconnected);
+
+  // The scheduler should be able to immediately reconnect with the master.
+  AWAIT_READY(connected);
+
+  EXPECT_CALL(*scheduler, disconnected(_))
+    .WillOnce(FutureSatisfy(&disconnected));
+
+  // Simulate a spurious master failure event at the scheduler.
+  detector->appoint(None());
+
+  AWAIT_READY(disconnected);
+
+  EXPECT_CALL(*scheduler, disconnected(_))
+    .Times(0);
+
+  EXPECT_CALL(*scheduler, connected(_))
+    .Times(0);
+
+  mesos.reconnect();
+
+  // Flush any possible remaining events. The mocked scheduler will fail if the
+  // reconnection attempt resulted in any additional callbacks after the
+  // scheduler has disconnected.
+  Clock::pause();
+  Clock::settle();
+}
+
+
 // TODO(benh): Write test for sending Call::Acknowledgement through
 // master to slave when Event::Update was generated locally.