You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/02/03 04:10:57 UTC
svn commit: r1239988 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/
Author: eli
Date: Fri Feb 3 03:10:57 2012
New Revision: 1239988
URL: http://svn.apache.org/viewvc?rev=1239988&view=rev
Log:
HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be marked required. Contributed by Aaron T. Myers.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1239988&r1=1239987&r2=1239988&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Fri Feb 3 03:10:57 2012
@@ -155,3 +155,6 @@ HDFS-2859. LOCAL_ADDRESS_MATCHER.match h
HDFS-2861. checkpointing should verify that the dfs.http.address has been configured to a non-loopback for peer NN (todd)
HDFS-2860. TestDFSRollback#testRollback is failing. (atm)
+
+HDFS-2769. HA: When HA is enabled with a shared edits dir, that dir should be
+marked required. (atm via eli)
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1239988&r1=1239987&r2=1239988&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Feb 3 03:10:57 2012
@@ -632,9 +632,19 @@ public class FSNamesystem implements Nam
public static Collection<URI> getNamespaceDirs(Configuration conf) {
return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY);
}
-
+
+ /**
+ * Get all edits dirs which are required. If any shared edits dirs are
+ * configured, these are also included in the set of required dirs.
+ *
+ * @param conf the HDFS configuration.
+ * @return all required dirs.
+ */
public static Collection<URI> getRequiredNamespaceEditsDirs(Configuration conf) {
- return getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY);
+ Set<URI> ret = new HashSet<URI>();
+ ret.addAll(getStorageDirs(conf, DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY));
+ ret.addAll(getSharedEditsDirs(conf));
+ return ret;
}
private static Collection<URI> getStorageDirs(Configuration conf,
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java?rev=1239988&r1=1239987&r2=1239988&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java Fri Feb 3 03:10:57 2012
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,26 +34,48 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
+import com.google.common.base.Joiner;
+
public class TestFailureOfSharedDir {
private static final Log LOG = LogFactory.getLog(TestFailureOfSharedDir.class);
/**
+ * Test that the shared edits dir is automatically added to the list of edits
+ * dirs that are marked required.
+ */
+ @Test
+ public void testSharedDirIsAutomaticallyMarkedRequired()
+ throws URISyntaxException {
+ URI foo = new URI("file:/foo");
+ URI bar = new URI("file:/bar");
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, Joiner.on(",").join(foo, bar));
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY, foo.toString());
+ assertFalse(FSNamesystem.getRequiredNamespaceEditsDirs(conf).contains(
+ bar));
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, bar.toString());
+ Collection<URI> requiredEditsDirs = FSNamesystem
+ .getRequiredNamespaceEditsDirs(conf);
+ assertTrue(Joiner.on(",").join(requiredEditsDirs) + " does not contain " + bar,
+ requiredEditsDirs.contains(bar));
+ }
+
+ /**
* Test that marking the shared edits dir as being "required" causes the NN to
* fail if that dir can't be accessed.
*/
@Test
public void testFailureOfSharedDir() throws Exception {
Configuration conf = new Configuration();
+ // The shared edits dir will automatically be marked required.
URI sharedEditsUri = MiniDFSCluster.formatSharedEditsDir(
new File(MiniDFSCluster.getBaseDirectory()), 0, 1);
- // Mark the shared edits dir required.
- conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,
- sharedEditsUri.toString());
MiniDFSCluster cluster = null;
try {