You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/07/11 02:25:37 UTC
[2/2] hbase git commit: Revert "HBASE-18216 [AMv2] Workaround for
HBASE-18152, corrupt procedure WAL including ADDENDUM"
Revert "HBASE-18216 [AMv2] Workaround for HBASE-18152, corrupt procedure WAL including ADDENDUM"
This reverts commit 8ce3f49f8034417734ef1a3e6c624a601a22ace9.
See HBASE-18340
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/844596e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/844596e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/844596e0
Branch: refs/heads/branch-1
Commit: 844596e09e7961f506ab2e8bb4aee5557e1b4a44
Parents: 5d710bb
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Jul 10 19:06:26 2017 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Jul 10 19:25:09 2017 -0700
----------------------------------------------------------------------
.../store/wal/ProcedureWALFormatReader.java | 93 +++++---------------
1 file changed, 24 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/844596e0/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
index 8349611..281292d 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
@@ -43,7 +43,7 @@ public class ProcedureWALFormatReader {
private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class);
// ==============================================================================================
- // We read the WALs in reverse order from the newest to the oldest.
+ // We read the WALs in reverse order. from the newest to the oldest.
// We have different entry types:
// - INIT: Procedure submitted by the user (also known as 'root procedure')
// - INSERT: Children added to the procedure <parentId>:[<childId>, ...]
@@ -52,8 +52,7 @@ public class ProcedureWALFormatReader {
//
// In the WAL we can find multiple times the same procedure as UPDATE or INSERT.
// We read the WAL from top to bottom, so every time we find an entry of the
- // same procedure, that will be the "latest" update (Caveat: with multiple threads writing
- // the store, this assumption does not hold).
+ // same procedure, that will be the "latest" update.
//
// We keep two in-memory maps:
// - localProcedureMap: is the map containing the entries in the WAL we are processing
@@ -65,7 +64,7 @@ public class ProcedureWALFormatReader {
//
// The WAL is append-only so the last procedure in the WAL is the one that
// was in execution at the time we crashed/closed the server.
- // Given that, the procedure replay order can be inferred by the WAL order.
+ // given that, the procedure replay order can be inferred by the WAL order.
//
// Example:
// WAL-2: [A, B, A, C, D]
@@ -78,7 +77,7 @@ public class ProcedureWALFormatReader {
// WAL-2 localProcedureMap.replayOrder is [D, C, A, B]
// WAL-1 localProcedureMap.replayOrder is [F, G]
//
- // Each time we reach the WAL-EOF, the "replayOrder" list is merged/appended in 'procedureMap'
+ // each time we reach the WAL-EOF, the "replayOrder" list is merged/appended in 'procedureMap'
// so using the example above we end up with: [D, C, A, B] + [F, G] as replay order.
//
// Fast Start: INIT/INSERT record and StackIDs
@@ -154,12 +153,12 @@ public class ProcedureWALFormatReader {
log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId());
procedureMap.mergeTail(localProcedureMap);
//if (hasFastStartSupport) {
- // TODO: Some procedure may be already runnables (see readInitEntry())
- // (we can also check the "update map" in the log trackers)
- // --------------------------------------------------
- //EntryIterator iter = procedureMap.fetchReady();
- //if (iter != null) loader.load(iter);
- // --------------------------------------------------
+ // TODO: Some procedure may be already runnables (see readInitEntry())
+ // (we can also check the "update map" in the log trackers)
+ // --------------------------------------------------
+ //EntryIterator iter = procedureMap.fetchReady();
+ //if (iter != null) loader.load(iter);
+ // --------------------------------------------------
//}
}
}
@@ -190,7 +189,7 @@ public class ProcedureWALFormatReader {
}
private void readInitEntry(final ProcedureWALEntry entry)
- throws IOException {
+ throws IOException {
assert entry.getProcedureCount() == 1 : "Expected only one procedure";
loadProcedure(entry, entry.getProcedure(0));
}
@@ -279,25 +278,12 @@ public class ProcedureWALFormatReader {
protected ProcedureProtos.Procedure proto;
protected boolean ready = false;
- public Entry(Entry hashNext) {
- this.hashNext = hashNext;
- }
-
- public long getProcId() {
- return proto.getProcId();
- }
-
- public long getParentId() {
- return proto.getParentId();
- }
-
- public boolean hasParent() {
- return proto.hasParentId();
- }
+ public Entry(Entry hashNext) { this.hashNext = hashNext; }
- public boolean isReady() {
- return ready;
- }
+ public long getProcId() { return proto.getProcId(); }
+ public long getParentId() { return proto.getParentId(); }
+ public boolean hasParent() { return proto.hasParentId(); }
+ public boolean isReady() { return ready; }
public boolean isCompleted() {
if (!hasParent()) {
@@ -417,17 +403,11 @@ public class ProcedureWALFormatReader {
public void add(ProcedureProtos.Procedure procProto) {
trackProcIds(procProto.getProcId());
Entry entry = addToMap(procProto.getProcId(), procProto.hasParentId());
- boolean newEntry = entry.proto == null;
- // We have seen procedure WALs where the entries are out of order; see HBASE-18152.
- // To compensate, only replace the Entry procedure if for sure this new procedure
- // is indeed an entry that came later. TODO: Fix the writing of procedure info so
- // it does not violate basic expectation, that WALs contain procedure changes going
- // from start to finish in sequence.
- if (newEntry || isIncreasing(entry.proto, procProto)) {
- entry.proto = procProto;
- }
+ boolean isNew = entry.proto == null;
+ entry.proto = procProto;
addToReplayList(entry);
- if(newEntry) {
+
+ if (isNew) {
if (procProto.hasParentId()) {
childUnlinkedHead = addToLinkList(entry, childUnlinkedHead);
} else {
@@ -436,21 +416,6 @@ public class ProcedureWALFormatReader {
}
}
- /**
- * @return True if this new procedure is 'richer' than the current one else
- * false and we log this incidence where it appears that the WAL has older entries
- * appended after newer ones. See HBASE-18152.
- */
- private static boolean isIncreasing(ProcedureProtos.Procedure current,
- ProcedureProtos.Procedure candidate) {
- boolean increasing = current.getStackIdCount() <= candidate.getStackIdCount() &&
- current.getLastUpdate() <= candidate.getLastUpdate();
- if (!increasing) {
- LOG.warn("NOT INCREASING! current=" + current + ", candidate=" + candidate);
- }
- return increasing;
- }
-
public boolean remove(long procId) {
trackProcIds(procId);
Entry entry = removeFromMap(procId);
@@ -627,7 +592,7 @@ public class ProcedureWALFormatReader {
}
/*
- * (see the comprehensive explanation in the beginning of the file)
+ * (see the comprehensive explaination in the beginning of the file)
* A Procedure is ready when parent and children are ready.
* "ready" means that we all the information that we need in-memory.
*
@@ -644,9 +609,9 @@ public class ProcedureWALFormatReader {
* - easy case, the parent is missing from the global map
* - more complex case we look at the Stack IDs
*
- * The Stack-IDs are added to the procedure order as an incremental index
+ * The Stack-IDs are added to the procedure order as incremental index
* tracking how many times that procedure was executed, which is equivalent
- * to the number of times we wrote the procedure to the WAL.
+ * at the number of times we wrote the procedure to the WAL.
* In the example above:
* wal-2: B has stackId = [1, 2]
* wal-1: B has stackId = [1]
@@ -688,10 +653,6 @@ public class ProcedureWALFormatReader {
int stackId = 1 + rootEntry.proto.getStackId(i);
maxStackId = Math.max(maxStackId, stackId);
stackIdSum += stackId;
- if (LOG.isTraceEnabled()) {
- LOG.trace("stackId=" + stackId + " stackIdSum=" + stackIdSum +
- " maxStackid=" + maxStackId + " " + rootEntry);
- }
}
for (Entry p = rootEntry.childHead; p != null; p = p.linkNext) {
@@ -699,14 +660,8 @@ public class ProcedureWALFormatReader {
int stackId = 1 + p.proto.getStackId(i);
maxStackId = Math.max(maxStackId, stackId);
stackIdSum += stackId;
- if (LOG.isTraceEnabled()) {
- LOG.trace("stackId=" + stackId + " stackIdSum=" + stackIdSum +
- " maxStackid=" + maxStackId + " " + p);
- }
}
}
- // The cmpStackIdSum is this formula for finding the sum of a series of numbers:
- // http://www.wikihow.com/Sum-the-Integers-from-1-to-N#/Image:Sum-the-Integers-from-1-to-N-Step-2-Version-3.jpg
final int cmpStackIdSum = (maxStackId * (maxStackId + 1) / 2);
if (cmpStackIdSum == stackIdSum) {
rootEntry.ready = true;
@@ -826,4 +781,4 @@ public class ProcedureWALFormatReader {
return (int)(Procedure.getProcIdHashCode(procId) % procedureMap.length);
}
}
-}
+}
\ No newline at end of file