You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2015/11/12 19:22:44 UTC
[32/50] [abbrv] hadoop git commit: HDFS-9249. NPE is thrown if an
IOException is thrown in NameNode constructor. (Wei-Chiu Chuang via Yongjun
Zhang)
HDFS-9249. NPE is thrown if an IOException is thrown in NameNode constructor. (Wei-Chiu Chuang via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2741a210
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2741a210
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2741a210
Branch: refs/heads/HDFS-8707
Commit: 2741a2109b98d0febb463cb318018ecbd3995102
Parents: 8fbea53
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Nov 9 14:04:03 2015 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Mon Nov 9 14:04:03 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 13 +++-
.../hdfs/server/namenode/TestBackupNode.java | 71 ++++++++++++++++++++
4 files changed, 88 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2741a210/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bf05c4e..533fe34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2277,6 +2277,9 @@ Release 2.8.0 - UNRELEASED
initialization, because HftpFileSystem is missing.
(Mingliang Liu via cnauroth)
+ HDFS-9249. NPE is thrown if an IOException is thrown in NameNode constructor.
+ (Wei-Chiu Chuang via Yongjun Zhang)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2741a210/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
index 3f9e1ad..36053f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
@@ -217,7 +217,9 @@ public class BackupNode extends NameNode {
// Abort current log segment - otherwise the NN shutdown code
// will close it gracefully, which is incorrect.
- getFSImage().getEditLog().abortCurrentLogSegment();
+ if (namesystem != null) {
+ getFSImage().getEditLog().abortCurrentLogSegment();
+ }
// Stop name-node threads
super.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2741a210/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 6e1f24f..7371d84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -889,15 +889,24 @@ public class NameNode implements NameNodeStatusMXBean {
haContext.writeUnlock();
}
} catch (IOException e) {
- this.stop();
+ this.stopAtException(e);
throw e;
} catch (HadoopIllegalArgumentException e) {
- this.stop();
+ this.stopAtException(e);
throw e;
}
this.started.set(true);
}
+ private void stopAtException(Exception e){
+ try {
+ this.stop();
+ } catch (Exception ex) {
+ LOG.warn("Encountered exception when handling exception ("
+ + e.getMessage() + "):", ex);
+ }
+ }
+
protected HAState createHAState(StartupOption startOpt) {
if (!haEnabled || startOpt == StartupOption.UPGRADE
|| startOpt == StartupOption.UPGRADEONLY) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2741a210/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
index 287a848..38d84e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -30,7 +31,9 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.directory.api.ldap.aci.UserClass;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -46,6 +49,8 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Before;
@@ -128,6 +133,72 @@ public class TestBackupNode {
Collections.singletonList((int)thisCheckpointTxId));
}
+
+ /**
+ * Regression test for HDFS-9249.
+ * This test configures the primary name node with SIMPLE authentication,
+ * and configures the backup node with Kerberose authentication with
+ * invalid keytab settings.
+ *
+ * This configuration causes the backup node to throw a NPE trying to abort
+ * the edit log.
+ * */
+ @Test
+ public void startBackupNodeWithIncorrectAuthentication() throws IOException {
+ Configuration c = new HdfsConfiguration();
+ StartupOption startupOpt = StartupOption.CHECKPOINT;
+ String dirs = getBackupNodeDir(startupOpt, 1);
+ c.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:1234");
+ c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY, "localhost:0");
+ c.set(DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY, "0");
+ c.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,
+ -1); // disable block scanner
+ c.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
+ c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
+ c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
+ "${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
+ c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
+ "127.0.0.1:0");
+ c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
+ "127.0.0.1:0");
+
+ NameNode nn;
+ try {
+ Configuration nnconf = new HdfsConfiguration(c);
+ DFSTestUtil.formatNameNode(nnconf);
+ nn = NameNode.createNameNode(new String[] {}, nnconf);
+ } catch (IOException e) {
+ LOG.info("IOException is thrown creating name node");
+ throw e;
+ }
+
+ c.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ c.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, "");
+
+ BackupNode bn = null;
+ try {
+ bn = (BackupNode)NameNode.createNameNode(
+ new String[] {startupOpt.getName()}, c);
+ assertTrue("Namesystem in BackupNode should be null",
+ bn.getNamesystem() == null);
+ fail("Incorrect authentication setting should throw IOException");
+ } catch (IOException e) {
+ LOG.info("IOException thrown as expected", e);
+ } finally {
+ if (nn != null) {
+ nn.stop();
+ }
+ if (bn != null) {
+ bn.stop();
+ }
+ SecurityUtil.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.SIMPLE, c);
+ // reset security authentication
+ UserGroupInformation.setConfiguration(c);
+ }
+ }
+
@Test
public void testCheckpointNode() throws Exception {
testCheckpoint(StartupOption.CHECKPOINT);