You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/11/08 18:22:50 UTC
[1/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Repository: hbase
Updated Branches:
refs/heads/branch-1 5fcfdcdc5 -> efc203a5a
refs/heads/branch-1.2 56ae22869 -> 3d9b0409c
refs/heads/branch-1.3 484b651fc -> 6bb7b4cde
refs/heads/branch-1.4 3f1fb46ca -> 58dfaab4f
refs/heads/branch-2 e5fb2f968 -> dcdebbffd
refs/heads/branch-2.0 5fff00419 -> 6214e7801
refs/heads/branch-2.1 0ec9f81bc -> 3a13088a2
refs/heads/master 2153d2c0c -> a8ad61ec8
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/efc203a5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/efc203a5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/efc203a5
Branch: refs/heads/branch-1
Commit: efc203a5a2d735723135f4f16b72075acb631ce0
Parents: 5fcfdcd
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Nov 7 18:24:13 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/efc203a5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 5643174..3edbc85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -336,6 +336,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -354,6 +355,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -370,13 +372,15 @@ public class ProtobufLogReader extends ReaderBase {
if (LOG.isTraceEnabled()) {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -404,16 +408,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[8/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a8ad61ec
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a8ad61ec
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a8ad61ec
Branch: refs/heads/master
Commit: a8ad61ec88f2d147e557f26543157db54dd7fcef
Parents: 2153d2c
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Nov 8 10:22:22 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8ad61ec/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 83398bd..494cce5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -340,6 +340,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -358,6 +359,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, ByteStreams.limit(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -375,13 +377,15 @@ public class ProtobufLogReader extends ReaderBase {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +
this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -409,16 +413,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[7/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a13088a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a13088a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a13088a
Branch: refs/heads/branch-2.1
Commit: 3a13088a2e01d7de13a63cbdfc4058e4d02e667e
Parents: 0ec9f81
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Nov 8 10:22:21 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3a13088a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 83398bd..494cce5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -340,6 +340,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -358,6 +359,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, ByteStreams.limit(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -375,13 +377,15 @@ public class ProtobufLogReader extends ReaderBase {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +
this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -409,16 +413,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[4/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3d9b0409
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3d9b0409
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3d9b0409
Branch: refs/heads/branch-1.2
Commit: 3d9b0409c3d86f84bb5d5e66cd6abe4e12735ea8
Parents: 56ae228
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Nov 7 18:24:31 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3d9b0409/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 9fd171f..73ad557 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -334,6 +334,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -352,6 +353,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -368,13 +370,15 @@ public class ProtobufLogReader extends ReaderBase {
if (LOG.isTraceEnabled()) {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -402,16 +406,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[3/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6bb7b4cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6bb7b4cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6bb7b4cd
Branch: refs/heads/branch-1.3
Commit: 6bb7b4cdeb105d30030af29495381c830f7b4540
Parents: 484b651f
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Nov 7 18:24:25 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bb7b4cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 9fd171f..73ad557 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -334,6 +334,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -352,6 +353,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -368,13 +370,15 @@ public class ProtobufLogReader extends ReaderBase {
if (LOG.isTraceEnabled()) {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -402,16 +406,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[6/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6214e780
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6214e780
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6214e780
Branch: refs/heads/branch-2.0
Commit: 6214e78018fb2d0ffbf6cc1265dfe1f7df0de574
Parents: 5fff004
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Nov 8 10:22:19 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6214e780/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 5d8d8c0..545bc21 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -338,6 +338,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -356,6 +357,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, ByteStreams.limit(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -373,13 +375,15 @@ public class ProtobufLogReader extends ReaderBase {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +
this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -407,16 +411,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[5/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dcdebbff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dcdebbff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dcdebbff
Branch: refs/heads/branch-2
Commit: dcdebbffdcd849eb8928eeccef72544e9a347371
Parents: e5fb2f9
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Nov 8 10:22:11 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcdebbff/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 83398bd..494cce5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -340,6 +340,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -358,6 +359,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, ByteStreams.limit(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -375,13 +377,15 @@ public class ProtobufLogReader extends ReaderBase {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +
this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -409,16 +413,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;
[2/8] hbase git commit: HBASE-20604 ProtobufLogReader#readNext can
incorrectly loop to the same position in the stream until the the WAL is
rolled
Posted by ap...@apache.org.
HBASE-20604 ProtobufLogReader#readNext can incorrectly loop to the same position in the stream until the the WAL is rolled
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58dfaab4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58dfaab4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58dfaab4
Branch: refs/heads/branch-1.4
Commit: 58dfaab4f6c985e7c51edba103eb69c51cd86af6
Parents: 3f1fb46
Author: Esteban Gutierrez <es...@apache.org>
Authored: Fri May 18 15:11:13 2018 -0500
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Nov 7 18:24:18 2018 -0800
----------------------------------------------------------------------
.../regionserver/wal/ProtobufLogReader.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/58dfaab4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index 5643174..3edbc85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -336,6 +336,7 @@ public class ProtobufLogReader extends ReaderBase {
}
WALKey.Builder builder = WALKey.newBuilder();
long size = 0;
+ boolean resetPosition = false;
try {
long available = -1;
try {
@@ -354,6 +355,7 @@ public class ProtobufLogReader extends ReaderBase {
ProtobufUtil.mergeFrom(builder, new LimitInputStream(this.inputStream, size),
(int)size);
} catch (InvalidProtocolBufferException ipbe) {
+ resetPosition = true;
throw (EOFException) new EOFException("Invalid PB, EOF? Ignoring; originalPosition=" +
originalPosition + ", currentPosition=" + this.inputStream.getPos() +
", messageSize=" + size + ", currentAvailable=" + available).initCause(ipbe);
@@ -370,13 +372,15 @@ public class ProtobufLogReader extends ReaderBase {
if (LOG.isTraceEnabled()) {
LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());
}
- continue;
+ seekOnFs(originalPosition);
+ return false;
}
int expectedCells = walKey.getFollowingKvCount();
long posBefore = this.inputStream.getPos();
try {
int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);
if (expectedCells != actualCells) {
+ resetPosition = true;
throw new EOFException("Only read " + actualCells); // other info added in catch
}
} catch (Exception ex) {
@@ -404,16 +408,28 @@ public class ProtobufLogReader extends ReaderBase {
// If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)
if (originalPosition < 0) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);
+ LOG.trace("Encountered a malformed edit, but can't seek back to last good position "
+ + "because originalPosition is negative. last offset="
+ + this.inputStream.getPos(), eof);
}
throw eof;
}
- // Else restore our position to original location in hope that next time through we will
- // read successfully.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);
+ // If stuck at the same place and we got and exception, lets go back at the beginning.
+ if (inputStream.getPos() == originalPosition && resetPosition) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking to the beginning of the WAL since "
+ + "current position and original position match at " + originalPosition);
+ }
+ seekOnFs(0);
+ } else {
+ // Else restore our position to original location in hope that next time through we will
+ // read successfully.
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Encountered a malformed edit, seeking back to last good position in file, "
+ + "from " + inputStream.getPos()+" to " + originalPosition, eof);
+ }
+ seekOnFs(originalPosition);
}
- seekOnFs(originalPosition);
return false;
}
return true;