You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/12/21 22:28:33 UTC
[7/8] mesos git commit: Added Log::Reader::catchup() method.
Added Log::Reader::catchup() method.
The new method catches-up positions missing in the local non-leading
replica to allow safe eventually consistent reads from it.
Review: https://reviews.apache.org/r/62287/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/49c91614
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/49c91614
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/49c91614
Branch: refs/heads/master
Commit: 49c91614da2a85dd49d1fdc33ef83f9c816de2d2
Parents: aa4d95a
Author: Ilya Pronin <ip...@twopensource.com>
Authored: Thu Dec 21 12:50:00 2017 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Thu Dec 21 12:50:00 2017 -0800
----------------------------------------------------------------------
include/mesos/log/log.hpp | 4 +++
src/log/log.cpp | 24 ++++++++++++++++++
src/log/log.hpp | 7 ++++++
src/tests/log_tests.cpp | 57 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 92 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/include/mesos/log/log.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/log/log.hpp b/include/mesos/log/log.hpp
index d92d7a0..ba35533 100644
--- a/include/mesos/log/log.hpp
+++ b/include/mesos/log/log.hpp
@@ -148,6 +148,10 @@ public:
// partitioned).
process::Future<Position> ending();
+ // Launches the catch-up process. Returns the ending position of
+ // the caught-up range.
+ process::Future<Position> catchup();
+
private:
internal::log::LogReaderProcess* process;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/log/log.cpp
----------------------------------------------------------------------
diff --git a/src/log/log.cpp b/src/log/log.cpp
index dcb66f7..6a7080a 100644
--- a/src/log/log.cpp
+++ b/src/log/log.cpp
@@ -36,6 +36,7 @@
#include <stout/nothing.hpp>
#include <stout/set.hpp>
+#include "log/catchup.hpp"
#include "log/coordinator.hpp"
#include "log/network.hpp"
#include "log/recover.hpp"
@@ -287,6 +288,8 @@ void LogProcess::discarded()
LogReaderProcess::LogReaderProcess(Log* log)
: ProcessBase(ID::generate("log-reader")),
+ quorum(log->process->quorum),
+ network(log->process->network),
recovering(dispatch(log->process, &LogProcess::recover)) {}
@@ -431,6 +434,21 @@ Future<list<Log::Entry>> LogReaderProcess::__read(
}
+Future<Log::Position> LogReaderProcess::catchup()
+{
+ return recover().then(defer(self(), &Self::_catchup));
+}
+
+
+Future<Log::Position> LogReaderProcess::_catchup()
+{
+ CHECK_READY(recovering);
+
+ return log::catchup(quorum, recovering.get(), network)
+ .then([](uint64_t end) { return Log::Position(end); });
+}
+
+
Log::Position LogReaderProcess::position(uint64_t value)
{
return Log::Position(value);
@@ -714,6 +732,12 @@ Future<Log::Position> Log::Reader::ending()
}
+Future<Log::Position> Log::Reader::catchup()
+{
+ return dispatch(process, &LogReaderProcess::catchup);
+}
+
+
/////////////////////////////////////////////////
// Public interfaces for Log::Writer.
/////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/log/log.hpp
----------------------------------------------------------------------
diff --git a/src/log/log.hpp b/src/log/log.hpp
index 24481aa..be038d1 100644
--- a/src/log/log.hpp
+++ b/src/log/log.hpp
@@ -128,6 +128,8 @@ public:
const mesos::log::Log::Position& from,
const mesos::log::Log::Position& to);
+ process::Future<mesos::log::Log::Position> catchup();
+
protected:
virtual void initialize();
virtual void finalize();
@@ -155,6 +157,11 @@ private:
const mesos::log::Log::Position& to,
const std::list<Action>& actions);
+ process::Future<mesos::log::Log::Position> _catchup();
+
+ const size_t quorum;
+ const process::Shared<Network> network;
+
process::Future<process::Shared<Replica>> recovering;
std::list<process::Promise<Nothing>*> promises;
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c91614/src/tests/log_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/log_tests.cpp b/src/tests/log_tests.cpp
index aada363..5dcb693 100644
--- a/src/tests/log_tests.cpp
+++ b/src/tests/log_tests.cpp
@@ -2384,6 +2384,63 @@ TEST_F(LogTest, Metrics)
}
+TEST_F(LogTest, ReaderCatchup)
+{
+ const string path1 = os::getcwd() + "/.log1";
+ initializer.flags.path = path1;
+ ASSERT_SOME(initializer.execute());
+
+ const string path2 = os::getcwd() + "/.log2";
+ initializer.flags.path = path2;
+ ASSERT_SOME(initializer.execute());
+
+ const string path3 = os::getcwd() + "/.log3";
+ initializer.flags.path = path3;
+ ASSERT_SOME(initializer.execute());
+
+ Shared<Replica> replica1(new Replica(path1));
+ Shared<Replica> replica2(new Replica(path2));
+
+ set<UPID> pids{replica1->pid(), replica2->pid()};
+ Shared<Network> network(new Network(pids));
+
+ Coordinator coord(2, replica2, network);
+
+ Future<Option<uint64_t>> electing = coord.elect();
+ AWAIT_READY(electing);
+ EXPECT_SOME_EQ(0u, electing.get());
+
+ // Add some entries to the log.
+ for (uint64_t position = 1; position <= 10; position++) {
+ Future<Option<uint64_t>> appending = coord.append(stringify(position));
+ AWAIT_READY(appending);
+ EXPECT_SOME_EQ(position, appending.get());
+ }
+
+ Log log3(2, path3, pids);
+ Log::Reader reader(&log3);
+
+ // Catch-up the replica that missed positions adding.
+ Future<Log::Position> end = reader.catchup();
+ AWAIT_READY(end);
+
+ Future<Log::Position> begin = reader.beginning();
+ AWAIT_READY(begin);
+
+ // We expect to read 9 entries instead of 10, because the catch-up
+ // procedure doesn't catch-up the last recovered position. See
+ // comments for RecoverMissingProcess.
+ Future<list<Log::Entry>> entries = reader.read(begin.get(), end.get());
+ AWAIT_READY(entries);
+ ASSERT_EQ(9u, entries->size());
+
+ uint64_t position = 1;
+ foreach (const Log::Entry& entry, entries.get()) {
+ EXPECT_EQ(stringify(position++), entry.data);
+ }
+}
+
+
#ifdef MESOS_HAS_JAVA
// TODO(jieyu): We copy the code from TemporaryDirectoryTest here
// because we cannot inherit from two test fixtures. In this future,