You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/01/05 23:50:28 UTC
mesos git commit: Add more logging into replicated log recovery.
Repository: mesos
Updated Branches:
refs/heads/master a54a7f629 -> 6409ab3e0
Add more logging into replicated log recovery.
Review: https://reviews.apache.org/r/28218
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6409ab3e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6409ab3e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6409ab3e
Branch: refs/heads/master
Commit: 6409ab3e052b3b26e4236221de686d6759ed35e1
Parents: a54a7f6
Author: Timothy Chen <tn...@apache.org>
Authored: Fri Dec 19 14:23:16 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Mon Jan 5 14:51:04 2015 -0800
----------------------------------------------------------------------
src/log/log.cpp | 4 ++++
src/log/recover.cpp | 12 ++++++++++++
src/log/replica.cpp | 3 +++
src/state/log.cpp | 9 +++++++++
4 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6409ab3e/src/log/log.cpp
----------------------------------------------------------------------
diff --git a/src/log/log.cpp b/src/log/log.cpp
index b3c6c20..598af69 100644
--- a/src/log/log.cpp
+++ b/src/log/log.cpp
@@ -339,6 +339,8 @@ void LogProcess::_recover()
Future<Owned<Replica> > future = recovering.get();
if (!future.isReady()) {
+ VLOG(2) << "Log recovery failed";
+
// The 'future' here can only be discarded in 'finalize'.
string failure = future.isFailed() ?
future.failure() :
@@ -353,6 +355,8 @@ void LogProcess::_recover()
}
promises.clear();
} else {
+ VLOG(2) << "Log recovery completed";
+
Owned<Replica> replica_ = future.get();
replica = replica_.share();
http://git-wip-us.apache.org/repos/asf/mesos/blob/6409ab3e/src/log/recover.cpp
----------------------------------------------------------------------
diff --git a/src/log/recover.cpp b/src/log/recover.cpp
index 2614065..679741b 100644
--- a/src/log/recover.cpp
+++ b/src/log/recover.cpp
@@ -128,6 +128,9 @@ private:
void start()
{
+ VLOG(2) << "Starting to wait for enough quorum of replicas before running "
+ << "recovery protocol, expected quroum size: " << stringify(quorum);
+
// Wait until there are enough (i.e., quorum of) replicas in the
// network to avoid unnecessary retries.
chain = network->watch(quorum, Network::GREATER_THAN_OR_EQUAL_TO)
@@ -139,6 +142,8 @@ private:
Future<Nothing> broadcast()
{
+ VLOG(2) << "Broadcasting recover request to all replicas";
+
// Broadcast recover request to all replicas.
return network->broadcast(protocol::recover, RecoverRequest())
.then(defer(self(), &Self::broadcasted, lambda::_1));
@@ -146,6 +151,8 @@ private:
Future<Nothing> broadcasted(const set<Future<RecoverResponse> >& _responses)
{
+ VLOG(2) << "Broadcast request completed";
+
responses = _responses;
// Reset the counters.
@@ -313,6 +320,8 @@ private:
promise.discard();
terminate(self());
} else {
+ VLOG(2) << "Log recovery timed out waiting for responses, retrying";
+
start(); // Re-run the recover protocol after timeout.
}
} else if (future.isFailed()) {
@@ -326,6 +335,9 @@ private:
// request while it is changing its status).
static const Duration T = Milliseconds(500);
Duration d = T * (1.0 + (double) ::random() / RAND_MAX);
+ VLOG(2) << "Didn't receive enough responses for recovery, retrying "
+ << "in " << stringify(d);
+
delay(d, self(), &Self::start);
} else {
promise.set(future.get().get());
http://git-wip-us.apache.org/repos/asf/mesos/blob/6409ab3e/src/log/replica.cpp
----------------------------------------------------------------------
diff --git a/src/log/replica.cpp b/src/log/replica.cpp
index c18de86..dedfde4 100644
--- a/src/log/replica.cpp
+++ b/src/log/replica.cpp
@@ -222,6 +222,9 @@ Future<list<Action> > ReplicaProcess::read(uint64_t from, uint64_t to)
return promise.future();
}
+ VLOG(2) << "Starting read from '" << stringify(from) << "' to '"
+ << stringify(to) << "'";
+
list<Action> actions;
for (uint64_t position = from; position <= to; position++) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/6409ab3e/src/state/log.cpp
----------------------------------------------------------------------
diff --git a/src/state/log.cpp b/src/state/log.cpp
index c0c81d5..326f3a7 100644
--- a/src/state/log.cpp
+++ b/src/state/log.cpp
@@ -221,6 +221,8 @@ Future<Nothing> LogStorageProcess::start()
return starting.get();
}
+ VLOG(2) << "Starting the writer";
+
starting = writer.start()
.then(defer(self(), &Self::_start, lambda::_1));
@@ -234,10 +236,15 @@ Future<Nothing> LogStorageProcess::_start(
CHECK_SOME(starting);
if (position.isNone()) {
+ VLOG(2) << "Writer failed to get elected, retrying";
+
starting = None(); // Reset 'starting' so we try again.
return start(); // TODO(benh): Don't try again forever?
}
+ VLOG(2) << "Writer got elected at position "
+ << position.get().identity();
+
// Now read and apply log entries. Since 'start' can be called
// multiple times (i.e., since we reset 'starting' after getting a
// None position returned after 'set', 'expunge', etc) we need to
@@ -277,6 +284,8 @@ Future<Nothing> LogStorageProcess::__start(
Future<Nothing> LogStorageProcess::apply(const list<Log::Entry>& entries)
{
+ VLOG(2) << "Applying operations (" << entries.size() << " entries)";
+
// Only read and apply entries past our index.
foreach (const Log::Entry& entry, entries) {
if (index.isNone() || index.get() < entry.position) {