You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/02/27 00:19:37 UTC
[1/3] hadoop git commit: HDFS-13145. SBN crash when transition to ANN
with in-progress edit tailing enabled. Contributed by Chao Sun.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 1fb87df87 -> 26395aef6
refs/heads/branch-3.1 9b37d1869 -> c58cec250
refs/heads/trunk b4f1ba141 -> ae290a4bb
HDFS-13145. SBN crash when transition to ANN with in-progress edit tailing enabled. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae290a4b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae290a4b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae290a4b
Branch: refs/heads/trunk
Commit: ae290a4bb4e514e2fe9b40d28426a7589afe2a3f
Parents: b4f1ba1
Author: Chao Sun <su...@apache.org>
Authored: Mon Feb 26 15:37:27 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Feb 26 16:15:00 2018 -0800
----------------------------------------------------------------------
.../qjournal/client/QuorumJournalManager.java | 4 ++-
.../client/TestQuorumJournalManager.java | 26 +++++++++++++++++++-
2 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae290a4b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 7dff9b4..7a70a3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -496,7 +496,9 @@ public class QuorumJournalManager implements JournalManager {
// If it's bounded by durable Txns, endTxId could not be larger
// than committedTxnId. This ensures the consistency.
- if (onlyDurableTxns && inProgressOk) {
+ // We don't do the following for finalized log segments, since all
+ // edits in those are guaranteed to be committed.
+ if (onlyDurableTxns && inProgressOk && remoteLog.isInProgress()) {
endTxId = Math.min(endTxId, committedTxnId);
if (endTxId < remoteLog.getStartTxId()) {
LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae290a4b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index ce1d404..34a0348 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -933,7 +933,31 @@ public class TestQuorumJournalManager {
verifyEdits(streams, 25, 50);
}
-
+
+ @Test
+ public void testInProgressRecovery() throws Exception {
+ // Test the case when in-progress edit log tailing is on, and
+ // new active performs recovery when the old active crashes
+ // without closing the last log segment.
+ // See HDFS-13145 for more details.
+
+ // Write two batches of edits. After these, the commitId on the
+ // journals should be 5, and endTxnId should be 8.
+ EditLogOutputStream stm = qjm.startLogSegment(1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 5);
+ writeTxns(stm, 6, 3);
+
+ // Do recovery from a separate QJM, just like in failover.
+ QuorumJournalManager qjm2 = createSpyingQJM();
+ qjm2.recoverUnfinalizedSegments();
+ checkRecovery(cluster, 1, 8);
+
+ // When selecting input stream, we should see all txns up to 8.
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm2.selectInputStreams(streams, 1, true, true);
+ verifyEdits(streams, 1, 8);
+ }
private QuorumJournalManager createSpyingQJM()
throws IOException, URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HDFS-13145. SBN crash when transition to ANN
with in-progress edit tailing enabled. Contributed by Chao Sun.
Posted by sh...@apache.org.
HDFS-13145. SBN crash when transition to ANN with in-progress edit tailing enabled. Contributed by Chao Sun.
(cherry picked from commit ae290a4bb4e514e2fe9b40d28426a7589afe2a3f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c58cec25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c58cec25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c58cec25
Branch: refs/heads/branch-3.1
Commit: c58cec25072e0c9fd714c4d5d8867f43894e9a19
Parents: 9b37d18
Author: Chao Sun <su...@apache.org>
Authored: Mon Feb 26 15:37:27 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Feb 26 16:17:36 2018 -0800
----------------------------------------------------------------------
.../qjournal/client/QuorumJournalManager.java | 4 ++-
.../client/TestQuorumJournalManager.java | 26 +++++++++++++++++++-
2 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c58cec25/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 7dff9b4..7a70a3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -496,7 +496,9 @@ public class QuorumJournalManager implements JournalManager {
// If it's bounded by durable Txns, endTxId could not be larger
// than committedTxnId. This ensures the consistency.
- if (onlyDurableTxns && inProgressOk) {
+ // We don't do the following for finalized log segments, since all
+ // edits in those are guaranteed to be committed.
+ if (onlyDurableTxns && inProgressOk && remoteLog.isInProgress()) {
endTxId = Math.min(endTxId, committedTxnId);
if (endTxId < remoteLog.getStartTxId()) {
LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c58cec25/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index ce1d404..34a0348 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -933,7 +933,31 @@ public class TestQuorumJournalManager {
verifyEdits(streams, 25, 50);
}
-
+
+ @Test
+ public void testInProgressRecovery() throws Exception {
+ // Test the case when in-progress edit log tailing is on, and
+ // new active performs recovery when the old active crashes
+ // without closing the last log segment.
+ // See HDFS-13145 for more details.
+
+ // Write two batches of edits. After these, the commitId on the
+ // journals should be 5, and endTxnId should be 8.
+ EditLogOutputStream stm = qjm.startLogSegment(1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 5);
+ writeTxns(stm, 6, 3);
+
+ // Do recovery from a separate QJM, just like in failover.
+ QuorumJournalManager qjm2 = createSpyingQJM();
+ qjm2.recoverUnfinalizedSegments();
+ checkRecovery(cluster, 1, 8);
+
+ // When selecting input stream, we should see all txns up to 8.
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm2.selectInputStreams(streams, 1, true, true);
+ verifyEdits(streams, 1, 8);
+ }
private QuorumJournalManager createSpyingQJM()
throws IOException, URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HDFS-13145. SBN crash when transition to ANN
with in-progress edit tailing enabled. Contributed by Chao Sun.
Posted by sh...@apache.org.
HDFS-13145. SBN crash when transition to ANN with in-progress edit tailing enabled. Contributed by Chao Sun.
(cherry picked from commit ae290a4bb4e514e2fe9b40d28426a7589afe2a3f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26395aef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26395aef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26395aef
Branch: refs/heads/branch-3.0
Commit: 26395aef674face8bf73feb91b88c2ae5fb9a400
Parents: 1fb87df
Author: Chao Sun <su...@apache.org>
Authored: Mon Feb 26 15:37:27 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Feb 26 16:18:51 2018 -0800
----------------------------------------------------------------------
.../qjournal/client/QuorumJournalManager.java | 4 ++-
.../client/TestQuorumJournalManager.java | 26 +++++++++++++++++++-
2 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26395aef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 7dff9b4..7a70a3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -496,7 +496,9 @@ public class QuorumJournalManager implements JournalManager {
// If it's bounded by durable Txns, endTxId could not be larger
// than committedTxnId. This ensures the consistency.
- if (onlyDurableTxns && inProgressOk) {
+ // We don't do the following for finalized log segments, since all
+ // edits in those are guaranteed to be committed.
+ if (onlyDurableTxns && inProgressOk && remoteLog.isInProgress()) {
endTxId = Math.min(endTxId, committedTxnId);
if (endTxId < remoteLog.getStartTxId()) {
LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26395aef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index ce1d404..34a0348 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -933,7 +933,31 @@ public class TestQuorumJournalManager {
verifyEdits(streams, 25, 50);
}
-
+
+ @Test
+ public void testInProgressRecovery() throws Exception {
+ // Test the case when in-progress edit log tailing is on, and
+ // new active performs recovery when the old active crashes
+ // without closing the last log segment.
+ // See HDFS-13145 for more details.
+
+ // Write two batches of edits. After these, the commitId on the
+ // journals should be 5, and endTxnId should be 8.
+ EditLogOutputStream stm = qjm.startLogSegment(1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ writeTxns(stm, 1, 5);
+ writeTxns(stm, 6, 3);
+
+ // Do recovery from a separate QJM, just like in failover.
+ QuorumJournalManager qjm2 = createSpyingQJM();
+ qjm2.recoverUnfinalizedSegments();
+ checkRecovery(cluster, 1, 8);
+
+ // When selecting input stream, we should see all txns up to 8.
+ List<EditLogInputStream> streams = new ArrayList<>();
+ qjm2.selectInputStreams(streams, 1, true, true);
+ verifyEdits(streams, 1, 8);
+ }
private QuorumJournalManager createSpyingQJM()
throws IOException, URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org