You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2019/08/23 19:29:41 UTC
[hadoop] branch branch-3.1 updated: HDFS-14674. [SBN read] Got an
unexpected txid when tail editlog. Contributed by wangzhaohui.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d2bb764 HDFS-14674. [SBN read] Got an unexpected txid when tail editlog. Contributed by wangzhaohui.
d2bb764 is described below
commit d2bb764ffa65d6f6ec806c040814263b46eeb5b3
Author: Chen Liang <cl...@apache.org>
AuthorDate: Fri Aug 23 12:17:34 2019 -0700
HDFS-14674. [SBN read] Got an unexpected txid when tail editlog. Contributed by wangzhaohui.
---
.../hadoop/hdfs/server/namenode/FSImage.java | 10 ++-
.../hadoop/hdfs/server/namenode/TestEditLog.java | 81 ++++++++++++++++++++++
2 files changed, 88 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 12e3101..469e7ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -891,7 +891,8 @@ public class FSImage implements Closeable {
StartupProgress prog = NameNode.getStartupProgress();
prog.beginPhase(Phase.LOADING_EDITS);
- long prevLastAppliedTxId = lastAppliedTxId;
+ long prevLastAppliedTxId = lastAppliedTxId;
+ long remainingReadTxns = maxTxnsToRead;
try {
FSEditLogLoader loader = new FSEditLogLoader(target, lastAppliedTxId);
@@ -908,8 +909,8 @@ public class FSImage implements Closeable {
(lastAppliedTxId + 1) + logSuppressed);
}
try {
- loader.loadFSEdits(editIn, lastAppliedTxId + 1, maxTxnsToRead,
- startOpt, recovery);
+ remainingReadTxns -= loader.loadFSEdits(editIn, lastAppliedTxId + 1,
+ remainingReadTxns, startOpt, recovery);
} finally {
// Update lastAppliedTxId even in case of error, since some ops may
// have been successfully applied before the error.
@@ -920,6 +921,9 @@ public class FSImage implements Closeable {
&& recovery != null) {
lastAppliedTxId = editIn.getLastTxId();
}
+ if (remainingReadTxns <= 0) {
+ break;
+ }
}
} finally {
FSEditLog.closeAllStreams(editStreams);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index b2524d3..ee02157 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -305,6 +305,87 @@ public class TestEditLog {
return loader.loadFSEdits(new EditLogByteInputStream(data), 1);
}
+ @Test
+ public void testMultiStreamsLoadEditWithConfMaxTxns()
+ throws IOException {
+ Configuration conf = getConf();
+ MiniDFSCluster cluster = null;
+ FileSystem fileSystem = null;
+ FSImage writeFsImage = null;
+ try {
+ cluster = new MiniDFSCluster
+ .Builder(conf)
+ .numDataNodes(NUM_DATA_NODES)
+ .build();
+ cluster.waitActive();
+ fileSystem = cluster.getFileSystem();
+ final FSNamesystem namesystem = cluster.getNamesystem();
+ writeFsImage = namesystem.getFSImage();
+ for (Iterator<URI> it = cluster.getNameDirs(0)
+ .iterator(); it.hasNext();) {
+ File dir = new File(it.next().getPath());
+ System.out.println(dir);
+ }
+ // Roll log so new output buffer size takes effect
+ // we should now be writing to edits_inprogress_3
+ long originalLastInodeId = namesystem.dir.getLastInodeId();
+ // Reopen some files as for append
+ Transactions trans = new Transactions(
+ namesystem, NUM_TRANSACTIONS, NUM_TRANSACTIONS / 2);
+ trans.run();
+ // Roll another time to finalize edits_inprogress_3
+ writeFsImage.rollEditLog(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ Transactions trans1 = new Transactions(
+ namesystem, NUM_TRANSACTIONS, NUM_TRANSACTIONS / 2);
+ trans1.run();
+ writeFsImage.rollEditLog(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ namesystem.dir.resetLastInodeIdWithoutChecking(originalLastInodeId);
+ for(Iterator<StorageDirectory> it = writeFsImage.getStorage().
+ dirIterator(NameNodeDirType.EDITS); it.hasNext();){
+ long expectedTxns = (2 * NUM_TRANSACTIONS) + 2;
+ File editFile = NNStorage.getFinalizedEditsFile(it.next(),
+ 1, expectedTxns);
+ File editFile1 = NNStorage.getFinalizedEditsFile(it.next(),
+ 203, 404);
+ assertTrue("Expect " + editFile + " exists", editFile.exists());
+ assertTrue("Expect " + editFile1 + " exists", editFile1.exists());
+ EditLogFileInputStream editLogFileInputStream1 =
+ new EditLogFileInputStream(editFile, 1, 202, false);
+ EditLogFileInputStream editLogFileInputStream2 =
+ new EditLogFileInputStream(editFile1, 203, 404, false);
+ List<EditLogInputStream> editStreams = Lists.newArrayList();
+ editStreams.add(editLogFileInputStream1);
+ editStreams.add(editLogFileInputStream2);
+ FSImage readFsImage = new FSImage(conf);
+ try {
+ readFsImage.loadEdits(editStreams, namesystem, 100, null, null);
+ } catch (Exception e){
+ LOG.error("There appears to be an out-of-order edit in the edit log",
+ e.getMessage());
+ fail("no exception should be thrown");
+ } finally {
+ if (readFsImage != null) {
+ readFsImage.close();
+ }
+ }
+ }
+ } finally {
+ try {
+ if(fileSystem != null) {
+ fileSystem.close();
+ }
+ if(cluster != null) {
+ cluster.shutdown();
+ }
+ if(writeFsImage != null){
+ writeFsImage.close();
+ }
+ } catch (Throwable t) {
+ LOG.error("Couldn't shut down cleanly", t);
+ }
+ }
+ }
+
/**
* Simple test for writing to and rolling the edit log.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org