You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/10/07 00:06:29 UTC
[25/50] [abbrv] hadoop git commit: HDFS-12420. Add an option to
disallow 'namenode format -force'. Contributed by Ajay Kumar.
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe
Branch: refs/heads/YARN-5734
Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75
Parents: d0046bb
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 5 15:26:52 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++
.../hadoop/hdfs/server/namenode/NameNode.java | 16 +++++++++
.../namenode/NameNodeFormatException.java | 37 ++++++++++++++++++++
.../src/main/resources/hdfs-default.xml | 10 ++++++
.../src/site/markdown/HDFSCommands.md | 2 +-
.../hdfs/server/namenode/TestClusterId.java | 34 ++++++++++++++++++
6 files changed, 101 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4842f9..f2f8730 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.posix.acl.inheritance.enabled";
public static final boolean
DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+ public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+ public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase implements
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
fsImage.getEditLog().initJournalsForWrite();
+ // Abort NameNode format if reformat is disabled and if
+ // meta-dir already exists
+ if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+ DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+ force = false;
+ isInteractive = false;
+ for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+ if (sd.hasSomeData()) {
+ throw new NameNodeFormatException(
+ "NameNode format aborted as reformat is disabled for "
+ + "this cluster.");
+ }
+ }
+ }
+
if (!fsImage.confirmFormat(force, isInteractive)) {
return true; // aborted
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
new file mode 100644
index 0000000..858d6e1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Thrown when NameNode format fails.
+ */
+@InterfaceAudience.Private
+public class NameNodeFormatException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public NameNodeFormatException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public NameNodeFormatException(String message) {
+ super(message);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4e4ee33..8a60b80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4640,4 +4640,14 @@
</description>
</property>
+ <property>
+ <name>dfs.reformat.disabled</name>
+ <value>false</value>
+ <description>
+ Disable reformat of NameNode. If it's value is set to "true"
+ and metadata directories already exist then attempt to format NameNode
+ will throw NameNodeFormatException.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 322609a..c5f80d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -532,7 +532,7 @@ Usage:
|:---- |:---- |
| `-backup` | Start backup node. |
| `-checkpoint` | Start checkpoint node. |
-| `-format` `[-clusterid cid]` `[-force]` `[-nonInteractive]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. -force option formats if the name directory exists. -nonInteractive option aborts if the name directory exists, unless -force option is specified. |
+| `-format` `[-clusterid cid]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. Will throw NameNodeFormatException if name dir already exist and if reformat is disabled for cluster. |
| `-upgrade` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Namenode should be started with upgrade option after the distribution of new Hadoop version. |
| `-upgradeOnly` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Upgrade the specified NameNode and then shutdown it. |
| `-rollback` | Rollback the NameNode to the previous version. This should be used after stopping the cluster and distributing the old Hadoop version. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
index fa3399b..d5a3948 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
@@ -39,9 +39,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ExitUtil.ExitException;
@@ -452,4 +456,34 @@ public class TestClusterId {
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
+
+ /**
+ * Test NameNode format failure when reformat is disabled and metadata
+ * directories exist.
+ */
+ @Test
+ public void testNNFormatFailure() throws Exception {
+ NameNode.initMetrics(config, NamenodeRole.NAMENODE);
+ DFSTestUtil.formatNameNode(config);
+ config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
+ // Call to NameNode format will fail as name dir is not empty
+ try {
+ NameNode.format(config);
+ fail("NN format should fail.");
+ } catch (NameNodeFormatException e) {
+ GenericTestUtils.assertExceptionContains("NameNode format aborted as "
+ + "reformat is disabled for this cluster", e);
+ }
+ }
+
+ /**
+ * Test NameNode format when reformat is disabled and metadata directories do
+ * not exist.
+ */
+ @Test
+ public void testNNFormatSuccess() throws Exception {
+ NameNode.initMetrics(config, NamenodeRole.NAMENODE);
+ config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
+ DFSTestUtil.formatNameNode(config);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org