You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/12/21 22:28:33 UTC

[7/8] mesos git commit: Added Log::Reader::catchup() method.

Added Log::Reader::catchup() method.

The new method catches-up positions missing in the local non-leading
replica to allow safe eventually consistent reads from it.

Review: https://reviews.apache.org/r/62287/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/49c91614
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/49c91614
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/49c91614

Branch: refs/heads/master
Commit: 49c91614da2a85dd49d1fdc33ef83f9c816de2d2
Parents: aa4d95a
Author: Ilya Pronin <ip...@twopensource.com>
Authored: Thu Dec 21 12:50:00 2017 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Thu Dec 21 12:50:00 2017 -0800

----------------------------------------------------------------------
 include/mesos/log/log.hpp |  4 +++
 src/log/log.cpp           | 24 ++++++++++++++++++
 src/log/log.hpp           |  7 ++++++
 src/tests/log_tests.cpp   | 57 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 92 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/include/mesos/log/log.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/log/log.hpp b/include/mesos/log/log.hpp
index d92d7a0..ba35533 100644
--- a/include/mesos/log/log.hpp
+++ b/include/mesos/log/log.hpp
@@ -148,6 +148,10 @@ public:
     // partitioned).
     process::Future<Position> ending();
 
+    // Launches the catch-up process. Returns the ending position of
+    // the caught-up range.
+    process::Future<Position> catchup();
+
   private:
     internal::log::LogReaderProcess* process;
   };

http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/log/log.cpp
----------------------------------------------------------------------
diff --git a/src/log/log.cpp b/src/log/log.cpp
index dcb66f7..6a7080a 100644
--- a/src/log/log.cpp
+++ b/src/log/log.cpp
@@ -36,6 +36,7 @@
 #include <stout/nothing.hpp>
 #include <stout/set.hpp>
 
+#include "log/catchup.hpp"
 #include "log/coordinator.hpp"
 #include "log/network.hpp"
 #include "log/recover.hpp"
@@ -287,6 +288,8 @@ void LogProcess::discarded()
 
 LogReaderProcess::LogReaderProcess(Log* log)
   : ProcessBase(ID::generate("log-reader")),
+    quorum(log->process->quorum),
+    network(log->process->network),
     recovering(dispatch(log->process, &LogProcess::recover)) {}
 
 
@@ -431,6 +434,21 @@ Future<list<Log::Entry>> LogReaderProcess::__read(
 }
 
 
+Future<Log::Position> LogReaderProcess::catchup()
+{
+  return recover().then(defer(self(), &Self::_catchup));
+}
+
+
+Future<Log::Position> LogReaderProcess::_catchup()
+{
+  CHECK_READY(recovering);
+
+  return log::catchup(quorum, recovering.get(), network)
+    .then([](uint64_t end) { return Log::Position(end); });
+}
+
+
 Log::Position LogReaderProcess::position(uint64_t value)
 {
   return Log::Position(value);
@@ -714,6 +732,12 @@ Future<Log::Position> Log::Reader::ending()
 }
 
 
+Future<Log::Position> Log::Reader::catchup()
+{
+  return dispatch(process, &LogReaderProcess::catchup);
+}
+
+
 /////////////////////////////////////////////////
 // Public interfaces for Log::Writer.
 /////////////////////////////////////////////////

http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/log/log.hpp
----------------------------------------------------------------------
diff --git a/src/log/log.hpp b/src/log/log.hpp
index 24481aa..be038d1 100644
--- a/src/log/log.hpp
+++ b/src/log/log.hpp
@@ -128,6 +128,8 @@ public:
       const mesos::log::Log::Position& from,
       const mesos::log::Log::Position& to);
 
+  process::Future<mesos::log::Log::Position> catchup();
+
 protected:
   virtual void initialize();
   virtual void finalize();
@@ -155,6 +157,11 @@ private:
       const mesos::log::Log::Position& to,
       const std::list<Action>& actions);
 
+  process::Future<mesos::log::Log::Position> _catchup();
+
+  const size_t quorum;
+  const process::Shared<Network> network;
+
   process::Future<process::Shared<Replica>> recovering;
   std::list<process::Promise<Nothing>*> promises;
 };

http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/tests/log_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/log_tests.cpp b/src/tests/log_tests.cpp
index aada363..5dcb693 100644
--- a/src/tests/log_tests.cpp
+++ b/src/tests/log_tests.cpp
@@ -2384,6 +2384,63 @@ TEST_F(LogTest, Metrics)
 }
 
 
+TEST_F(LogTest, ReaderCatchup)
+{
+  const string path1 = os::getcwd() + "/.log1";
+  initializer.flags.path = path1;
+  ASSERT_SOME(initializer.execute());
+
+  const string path2 = os::getcwd() + "/.log2";
+  initializer.flags.path = path2;
+  ASSERT_SOME(initializer.execute());
+
+  const string path3 = os::getcwd() + "/.log3";
+  initializer.flags.path = path3;
+  ASSERT_SOME(initializer.execute());
+
+  Shared<Replica> replica1(new Replica(path1));
+  Shared<Replica> replica2(new Replica(path2));
+
+  set<UPID> pids{replica1->pid(), replica2->pid()};
+  Shared<Network> network(new Network(pids));
+
+  Coordinator coord(2, replica2, network);
+
+  Future<Option<uint64_t>> electing = coord.elect();
+  AWAIT_READY(electing);
+  EXPECT_SOME_EQ(0u, electing.get());
+
+  // Add some entries to the log.
+  for (uint64_t position = 1; position <= 10; position++) {
+    Future<Option<uint64_t>> appending = coord.append(stringify(position));
+    AWAIT_READY(appending);
+    EXPECT_SOME_EQ(position, appending.get());
+  }
+
+  Log log3(2, path3, pids);
+  Log::Reader reader(&log3);
+
+  // Catch-up the replica that missed positions adding.
+  Future<Log::Position> end = reader.catchup();
+  AWAIT_READY(end);
+
+  Future<Log::Position> begin = reader.beginning();
+  AWAIT_READY(begin);
+
+  // We expect to read 9 entries instead of 10, because the catch-up
+  // procedure doesn't catch-up the last recovered position. See
+  // comments for RecoverMissingProcess.
+  Future<list<Log::Entry>> entries = reader.read(begin.get(), end.get());
+  AWAIT_READY(entries);
+  ASSERT_EQ(9u, entries->size());
+
+  uint64_t position = 1;
+  foreach (const Log::Entry& entry, entries.get()) {
+    EXPECT_EQ(stringify(position++), entry.data);
+  }
+}
+
+
 #ifdef MESOS_HAS_JAVA
 // TODO(jieyu): We copy the code from TemporaryDirectoryTest here
 // because we cannot inherit from two test fixtures. In this future,