You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/12/04 16:16:53 UTC
[02/50] [abbrv] hadoop git commit: HDFS-14075. Terminate the namenode
when failed to start log segment. Contributed by Ayush Saxena.
HDFS-14075. Terminate the namenode when failed to start log segment. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042c8ef5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042c8ef5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042c8ef5
Branch: refs/heads/HDDS-4
Commit: 042c8ef593ced1915a688e99aa9a6a52fdf66734
Parents: 8f3e12f
Author: Surendra Singh Lilhore <su...@apache.org>
Authored: Sun Dec 2 12:31:08 2018 +0530
Committer: Surendra Singh Lilhore <su...@apache.org>
Committed: Sun Dec 2 12:31:08 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +++--
.../hadoop/hdfs/server/namenode/JournalSet.java | 10 ++++-
.../hadoop/hdfs/qjournal/TestNNWithQJM.java | 15 +++----
.../hdfs/server/namenode/TestEditLog.java | 8 +++-
.../namenode/TestEditLogJournalFailures.java | 44 ++++++++++++++++++++
5 files changed, 74 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 547ad57..56aa927 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -1382,10 +1382,15 @@ public class FSEditLog implements LogsPurgeable {
try {
editLogStream = journalSet.startLogSegment(segmentTxId, layoutVersion);
} catch (IOException ex) {
- throw new IOException("Unable to start log segment " +
- segmentTxId + ": too few journals successfully started.", ex);
+ final String msg = "Unable to start log segment " + segmentTxId
+ + ": too few journals successfully started.";
+ LOG.error(msg, ex);
+ synchronized (journalSetLock) {
+ IOUtils.cleanupWithLogger(LOG, journalSet);
+ }
+ terminate(1, msg);
}
-
+
curSegmentTxId = segmentTxId;
state = State.IN_SEGMENT;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
index 7be7073..4ab0828 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
@@ -76,7 +77,7 @@ public class JournalSet implements JournalManager {
* stream, then the stream will be aborted and set to null.
*/
static class JournalAndStream implements CheckableNameNodeResource {
- private final JournalManager journal;
+ private JournalManager journal;
private boolean disabled = false;
private EditLogOutputStream stream;
private final boolean required;
@@ -146,7 +147,12 @@ public class JournalSet implements JournalManager {
void setCurrentStreamForTests(EditLogOutputStream stream) {
this.stream = stream;
}
-
+
+ @VisibleForTesting
+ void setJournalForTests(JournalManager jm) {
+ this.journal = jm;
+ }
+
JournalManager getManager() {
return journal;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
index d713bc7..4483667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -191,15 +192,15 @@ public class TestNNWithQJM {
// Start the NN - should fail because the JNs are still formatted
// with the old namespace ID.
try {
- cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(0)
- .manageNameDfsDirs(false)
- .format(false)
- .build();
+ ExitUtil.disableSystemExit();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false)
+ .build();
fail("New NN with different namespace should have been rejected");
- } catch (IOException ioe) {
+ } catch (ExitException ee) {
GenericTestUtils.assertExceptionContains(
- "Unable to start log segment 1: too few journals", ioe);
+ "Unable to start log segment 1: too few journals", ee);
+ assertTrue("Didn't terminate properly ", ExitUtil.terminateCalled());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index d24bffa..8eac143 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -81,6 +81,8 @@ import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
@@ -974,17 +976,19 @@ public class TestEditLog {
public void testFailedOpen() throws Exception {
File logDir = new File(TEST_DIR, "testFailedOpen");
logDir.mkdirs();
+ ExitUtil.disableSystemExit();
FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
try {
FileUtil.setWritable(logDir, false);
log.openForWrite(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
fail("Did no throw exception on only having a bad dir");
- } catch (IOException ioe) {
+ } catch (ExitException ee) {
GenericTestUtils.assertExceptionContains(
- "too few journals successfully started", ioe);
+ "too few journals successfully started", ee);
} finally {
FileUtil.setWritable(logDir, true);
log.close();
+ ExitUtil.resetFirstExitException();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
index 1e8ee9c..07791da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
@@ -34,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
@@ -262,6 +266,46 @@ public class TestEditLogJournalFailures {
}
}
+ @Test
+ public void testMultipleRedundantFailedEditsDirOnStartLogSegment()
+ throws Exception {
+ // Set up 4 name/edits dirs.
+ shutDownMiniCluster();
+ Configuration conf = getConf();
+ String[] nameDirs = new String[4];
+ for (int i = 0; i < nameDirs.length; i++) {
+ File nameDir = new File(PathUtils.getTestDir(getClass()), "name-dir" + i);
+ nameDir.mkdirs();
+ nameDirs[i] = nameDir.getAbsolutePath();
+ }
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ StringUtils.join(nameDirs, ","));
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,
+ StringUtils.join(nameDirs, ",", 0, 3));
+
+ setUpMiniCluster(conf, false);
+
+ // All journals active.
+ assertTrue(doAnEdit());
+ // The NN has not terminated (no ExitException thrown)
+ spyOnJASjournal(3);
+ RemoteException re = intercept(RemoteException.class,
+ "too few journals successfully started.",
+ () -> ((DistributedFileSystem) fs).rollEdits());
+ GenericTestUtils.assertExceptionContains("ExitException", re);
+ }
+
+ private JournalManager spyOnJASjournal(int index) throws Exception {
+ JournalAndStream jas = getJournalAndStream(index);
+ JournalManager manager = jas.getManager();
+ JournalManager spyManager = spy(manager);
+ jas.setJournalForTests(spyManager);
+ doThrow(new IOException("Unable to start log segment ")).when(spyManager)
+ .startLogSegment(anyLong(), anyInt());
+
+ return spyManager;
+ }
+
/**
* Replace the journal at index <code>index</code> with one that throws an
* exception on flush.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org