You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/04/11 00:22:30 UTC
svn commit: r1312024 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/common/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: todd
Date: Tue Apr 10 22:22:29 2012
New Revision: 1312024
URL: http://svn.apache.org/viewvc?rev=1312024&view=rev
Log:
HDFS-3094. add -nonInteractive and -force option to namenode -format command. Contributed by Arpit Gupta.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1312024&r1=1312023&r2=1312024&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Apr 10 22:22:29 2012
@@ -231,6 +231,9 @@ Release 2.0.0 - UNRELEASED
HDFS-3238. ServerCommand and friends don't need to be writables. (eli)
+ HDFS-3094. add -nonInteractive and -force option to namenode -format
+ command (Arpit Gupta via todd)
+
OPTIMIZATIONS
HDFS-2477. Optimize computing the diff between a block report and the
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java?rev=1312024&r1=1312023&r2=1312024&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java Tue Apr 10 22:22:29 2012
@@ -57,13 +57,18 @@ public final class HdfsServerConstants {
BOOTSTRAPSTANDBY("-bootstrapStandby"),
INITIALIZESHAREDEDITS("-initializeSharedEdits"),
RECOVER ("-recover"),
- FORCE("-force");
+ FORCE("-force"),
+ NONINTERACTIVE("-nonInteractive");
private String name = null;
// Used only with format and upgrade options
private String clusterId = null;
+ // Used only with format option
+ private boolean isForceFormat = false;
+ private boolean isInteractiveFormat = true;
+
// Used only with recovery option
private int force = 0;
@@ -101,6 +106,22 @@ public final class HdfsServerConstants {
public int getForce() {
return this.force;
}
+
+ public boolean getForceFormat() {
+ return isForceFormat;
+ }
+
+ public void setForceFormat(boolean force) {
+ isForceFormat = force;
+ }
+
+ public boolean getInteractiveFormat() {
+ return isInteractiveFormat;
+ }
+
+ public void setInteractiveFormat(boolean interactive) {
+ isInteractiveFormat = interactive;
+ }
}
// Timeouts for communicating with DataNode for streaming writes/reads
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1312024&r1=1312023&r2=1312024&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Apr 10 22:22:29 2012
@@ -215,7 +215,7 @@ public class NameNode {
/** Format a new filesystem. Destroys any filesystem that may already
* exist at this location. **/
public static void format(Configuration conf) throws IOException {
- format(conf, true);
+ format(conf, true, true);
}
static NameNodeMetrics metrics;
@@ -658,9 +658,8 @@ public class NameNode {
* @return true if formatting was aborted, false otherwise
* @throws IOException
*/
- private static boolean format(Configuration conf,
- boolean force)
- throws IOException {
+ private static boolean format(Configuration conf, boolean force,
+ boolean isInteractive) throws IOException {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
@@ -669,7 +668,7 @@ public class NameNode {
Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
List<URI> editDirsToFormat =
FSNamesystem.getNamespaceEditsDirs(conf);
- if (!confirmFormat(dirsToFormat, force, true)) {
+ if (!confirmFormat(dirsToFormat, force, isInteractive)) {
return true; // aborted
}
@@ -830,8 +829,9 @@ public class NameNode {
"Usage: java NameNode [" +
StartupOption.BACKUP.getName() + "] | [" +
StartupOption.CHECKPOINT.getName() + "] | [" +
- StartupOption.FORMAT.getName() + "[" + StartupOption.CLUSTERID.getName() +
- " cid ]] | [" +
+ StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +
+ " cid ] [" + StartupOption.FORCE.getName() + "] [" +
+ StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
StartupOption.UPGRADE.getName() + "] | [" +
StartupOption.ROLLBACK.getName() + "] | [" +
StartupOption.FINALIZE.getName() + "] | [" +
@@ -850,11 +850,35 @@ public class NameNode {
String cmd = args[i];
if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.FORMAT;
- // might be followed by two args
- if (i + 2 < argsLen
- && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
- i += 2;
- startOpt.setClusterId(args[i]);
+ for (i = i + 1; i < argsLen; i++) {
+ if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
+ i++;
+ if (i >= argsLen) {
+ // if no cluster id specified, return null
+ LOG.fatal("Must specify a valid cluster ID after the "
+ + StartupOption.CLUSTERID.getName() + " flag");
+ return null;
+ }
+ String clusterId = args[i];
+ // Make sure an id is specified and not another flag
+ if (clusterId.isEmpty() ||
+ clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) ||
+ clusterId.equalsIgnoreCase(
+ StartupOption.NONINTERACTIVE.getName())) {
+ LOG.fatal("Must specify a valid cluster ID after the "
+ + StartupOption.CLUSTERID.getName() + " flag");
+ return null;
+ }
+ startOpt.setClusterId(clusterId);
+ }
+
+ if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
+ startOpt.setForceFormat(true);
+ }
+
+ if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
+ startOpt.setInteractiveFormat(false);
+ }
}
} else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.GENCLUSTERID;
@@ -997,7 +1021,8 @@ public class NameNode {
switch (startOpt) {
case FORMAT: {
- boolean aborted = format(conf, false);
+ boolean aborted = format(conf, startOpt.getForceFormat(),
+ startOpt.getInteractiveFormat());
System.exit(aborted ? 1 : 0);
return null; // avoid javac warning
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java?rev=1312024&r1=1312023&r2=1312024&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java Tue Apr 10 22:22:29 2012
@@ -18,12 +18,19 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
import java.net.URI;
-import java.util.ArrayList;
+import java.security.Permission;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -40,11 +47,11 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
public class TestClusterId {
private static final Log LOG = LogFactory.getLog(TestClusterId.class);
File hdfsDir;
-
+ Configuration config;
+
private String getClusterId(Configuration config) throws IOException {
// see if cluster id not empty.
Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(config);
@@ -59,33 +66,41 @@ public class TestClusterId {
LOG.info("successfully formated : sd="+sd.getCurrentDir() + ";cid="+cid);
return cid;
}
-
+
@Before
public void setUp() throws IOException {
+ System.setSecurityManager(new NoExitSecurityManager());
+
String baseDir = System.getProperty("test.build.data", "build/test/data");
- hdfsDir = new File(baseDir, "dfs");
- if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
- throw new IOException("Could not delete test directory '" +
- hdfsDir + "'");
+ hdfsDir = new File(baseDir, "dfs/name");
+ if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+ throw new IOException("Could not delete test directory '" + hdfsDir + "'");
}
LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath());
+
+ // as some tests might change these values we reset them to defaults before
+ // every test
+ StartupOption.FORMAT.setForceFormat(false);
+ StartupOption.FORMAT.setInteractiveFormat(true);
+
+ config = new Configuration();
+ config.set(DFS_NAMENODE_NAME_DIR_KEY, hdfsDir.getPath());
}
-
+
@After
public void tearDown() throws IOException {
- if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
- throw new IOException("Could not tearDown test directory '" +
- hdfsDir + "'");
+ System.setSecurityManager(null);
+
+ if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
+ throw new IOException("Could not tearDown test directory '" + hdfsDir
+ + "'");
}
}
-
+
@Test
public void testFormatClusterIdOption() throws IOException {
- Configuration config = new Configuration();
- config.set(DFS_NAMENODE_NAME_DIR_KEY, new File(hdfsDir, "name").getPath());
-
// 1. should format without cluster id
//StartupOption.FORMAT.setClusterId("");
NameNode.format(config);
@@ -107,4 +122,356 @@ public class TestClusterId {
String newCid = getClusterId(config);
assertFalse("ClusterId should not be the same", newCid.equals(cid));
}
-}
+
+ /**
+ * Test namenode format with -format option. Format should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormat() throws IOException {
+ String[] argv = { "-format" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -format option when an empty name directory
+ * exists. Format should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithEmptyDir() throws IOException {
+
+ if (!hdfsDir.mkdirs()) {
+ fail("Failed to create dir " + hdfsDir.getPath());
+ }
+
+ String[] argv = { "-format" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -format -force options when name directory
+ * exists. Format should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithForce() throws IOException {
+
+ if (!hdfsDir.mkdirs()) {
+ fail("Failed to create dir " + hdfsDir.getPath());
+ }
+
+ String[] argv = { "-format", "-force" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -format -force -clusterid option when name
+ * directory exists. Format should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithForceAndClusterId() throws IOException {
+
+ if (!hdfsDir.mkdirs()) {
+ fail("Failed to create dir " + hdfsDir.getPath());
+ }
+
+ String myId = "testFormatWithForceAndClusterId";
+ String[] argv = { "-format", "-force", "-clusterid", myId };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cId = getClusterId(config);
+ assertEquals("ClusterIds do not match", myId, cId);
+ }
+
+ /**
+ * Test namenode format with -clusterid -force option. Format command should
+ * fail as no cluster id was provided.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithInvalidClusterIdOption() throws IOException {
+
+ String[] argv = { "-format", "-clusterid", "-force" };
+ PrintStream origErr = System.err;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream stdErr = new PrintStream(baos);
+ System.setErr(stdErr);
+
+ NameNode.createNameNode(argv, config);
+
+ // Check if usage is printed
+ assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+ System.setErr(origErr);
+
+ // check if the version file does not exists.
+ File version = new File(hdfsDir, "current/VERSION");
+ assertFalse("Check version should not exist", version.exists());
+ }
+
+ /**
+ * Test namenode format with -format -clusterid options. Format should fail
+ * was no clusterid was sent.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithNoClusterIdOption() throws IOException {
+
+ String[] argv = { "-format", "-clusterid" };
+ PrintStream origErr = System.err;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream stdErr = new PrintStream(baos);
+ System.setErr(stdErr);
+
+ NameNode.createNameNode(argv, config);
+
+ // Check if usage is printed
+ assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+ System.setErr(origErr);
+
+ // check if the version file does not exists.
+ File version = new File(hdfsDir, "current/VERSION");
+ assertFalse("Check version should not exist", version.exists());
+ }
+
+ /**
+ * Test namenode format with -format -clusterid and empty clusterid. Format
+ * should fail as no valid if was provided.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithEmptyClusterIdOption() throws IOException {
+
+ String[] argv = { "-format", "-clusterid", "" };
+
+ PrintStream origErr = System.err;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream stdErr = new PrintStream(baos);
+ System.setErr(stdErr);
+
+ NameNode.createNameNode(argv, config);
+
+ // Check if usage is printed
+ assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+ System.setErr(origErr);
+
+ // check if the version file does not exists.
+ File version = new File(hdfsDir, "current/VERSION");
+ assertFalse("Check version should not exist", version.exists());
+ }
+
+ /**
+ * Test namenode format with -format -nonInteractive options when a non empty
+ * name directory exists. Format should not succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithNonInteractive() throws IOException {
+
+ // we check for a non empty dir, so create a child path
+ File data = new File(hdfsDir, "file");
+ if (!data.mkdirs()) {
+ fail("Failed to create dir " + data.getPath());
+ }
+
+ String[] argv = { "-format", "-nonInteractive" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have been aborted with exit code 1", 1,
+ e.status);
+ }
+
+ // check if the version file does not exists.
+ File version = new File(hdfsDir, "current/VERSION");
+ assertFalse("Check version should not exist", version.exists());
+ }
+
+ /**
+ * Test namenode format with -format -nonInteractive options when name
+ * directory does not exist. Format should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithNonInteractiveNameDirDoesNotExit()
+ throws IOException {
+
+ String[] argv = { "-format", "-nonInteractive" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -force -nonInteractive -force option. Format
+ * should succeed.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testFormatWithNonInteractiveAndForce() throws IOException {
+
+ if (!hdfsDir.mkdirs()) {
+ fail("Failed to create dir " + hdfsDir.getPath());
+ }
+
+ String[] argv = { "-format", "-nonInteractive", "-force" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -format option when a non empty name directory
+ * exists. Enter Y when prompted and the format should succeed.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testFormatWithoutForceEnterYes() throws IOException,
+ InterruptedException {
+
+ // we check for a non empty dir, so create a child path
+ File data = new File(hdfsDir, "file");
+ if (!data.mkdirs()) {
+ fail("Failed to create dir " + data.getPath());
+ }
+
+ // capture the input stream
+ InputStream origIn = System.in;
+ ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes());
+ System.setIn(bins);
+
+ String[] argv = { "-format" };
+
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should have succeeded", 0, e.status);
+ }
+
+ System.setIn(origIn);
+
+ String cid = getClusterId(config);
+ assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+ }
+
+ /**
+ * Test namenode format with -format option when a non empty name directory
+ * exists. Enter N when prompted and format should be aborted.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testFormatWithoutForceEnterNo() throws IOException,
+ InterruptedException {
+
+ // we check for a non empty dir, so create a child path
+ File data = new File(hdfsDir, "file");
+ if (!data.mkdirs()) {
+ fail("Failed to create dir " + data.getPath());
+ }
+
+ // capture the input stream
+ InputStream origIn = System.in;
+ ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes());
+ System.setIn(bins);
+
+ String[] argv = { "-format" };
+ try {
+ NameNode.createNameNode(argv, config);
+ fail("createNameNode() did not call System.exit()");
+ } catch (ExitException e) {
+ assertEquals("Format should not have succeeded", 1, e.status);
+ }
+
+ System.setIn(origIn);
+
+ // check if the version file does not exists.
+ File version = new File(hdfsDir, "current/VERSION");
+ assertFalse("Check version should not exist", version.exists());
+ }
+
+ private static class ExitException extends SecurityException {
+ private static final long serialVersionUID = 1L;
+ public final int status;
+
+ public ExitException(int status) {
+ super("There is no escape!");
+ this.status = status;
+ }
+ }
+
+ private static class NoExitSecurityManager extends SecurityManager {
+ @Override
+ public void checkPermission(Permission perm) {
+ // allow anything.
+ }
+
+ @Override
+ public void checkPermission(Permission perm, Object context) {
+ // allow anything.
+ }
+
+ @Override
+ public void checkExit(int status) {
+ super.checkExit(status);
+ throw new ExitException(status);
+ }
+ }
+}
\ No newline at end of file