You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/02/06 09:24:13 UTC
svn commit: r1240916 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/
src/test/java/org/apache/hadoop/hdfs/server/namenode/ src/test/j...
Author: eli
Date: Mon Feb 6 08:24:11 2012
New Revision: 1240916
URL: http://svn.apache.org/viewvc?rev=1240916&view=rev
Log:
HDFS-2752. HA: exit if multiple shared dirs are configured. Contributed by Eli Collins
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Mon Feb 6 08:24:11 2012
@@ -170,3 +170,5 @@ HDFS-2792. Make fsck work. (atm)
HDFS-2808. HA: haadmin should use namenode ids. (eli)
HDFS-2819. Document new HA-related configs in hdfs-default.xml. (eli)
+
+HDFS-2752. HA: exit if multiple shared dirs are configured. (eli)
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Feb 6 08:24:11 2012
@@ -683,17 +683,28 @@ public class FSNamesystem implements Nam
* are ordered before non-shared directories, and any duplicates
* are removed. The order they are specified in the configuration
* is retained.
+ * @return Collection of shared edits directories.
+ * @throws IOException if multiple shared edits directories are configured
*/
- public static List<URI> getNamespaceEditsDirs(Configuration conf) {
+ public static List<URI> getNamespaceEditsDirs(Configuration conf)
+ throws IOException {
// Use a LinkedHashSet so that order is maintained while we de-dup
// the entries.
LinkedHashSet<URI> editsDirs = new LinkedHashSet<URI>();
+ List<URI> sharedDirs = getSharedEditsDirs(conf);
+
+ // Fail until multiple shared edits directories are supported (HDFS-2782)
+ if (sharedDirs.size() > 1) {
+ throw new IOException(
+ "Multiple shared edits directories are not yet supported");
+ }
+
// First add the shared edits dirs. It's critical that the shared dirs
// are added first, since JournalSet syncs them in the order they are listed,
// and we need to make sure all edits are in place in the shared storage
// before they are replicated locally. See HDFS-2874.
- for (URI dir : getSharedEditsDirs(conf)) {
+ for (URI dir : sharedDirs) {
if (!editsDirs.add(dir)) {
LOG.warn("Edits URI " + dir + " listed multiple times in " +
DFS_NAMENODE_SHARED_EDITS_DIR_KEY + ". Ignoring duplicates.");
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Mon Feb 6 08:24:11 2012
@@ -1594,7 +1594,7 @@ public class MiniDFSCluster {
/**
* Get the directories where the namenode stores its edits.
*/
- public Collection<URI> getNameEditsDirs(int nnIndex) {
+ public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException {
return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java Mon Feb 6 08:24:11 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.na
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
+import java.io.IOException;
import java.net.URI;
import java.util.Collection;
@@ -33,7 +34,7 @@ public class TestFSNamesystem {
* Tests that the namenode edits dirs are gotten with duplicates removed
*/
@Test
- public void testUniqueEditDirs() {
+ public void testUniqueEditDirs() throws IOException {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureOfSharedDir.java Mon Feb 6 08:24:11 2012
@@ -68,6 +68,29 @@ public class TestFailureOfSharedDir {
requiredEditsDirs.contains(bar));
}
+ /**
+ * Multiple shared edits directories is an invalid configuration.
+ */
+ @Test
+ public void testMultipleSharedDirsFails() throws Exception {
+ Configuration conf = new Configuration();
+ URI sharedA = new URI("file:///shared-A");
+ URI sharedB = new URI("file:///shared-B");
+ URI localA = new URI("file:///local-A");
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
+ Joiner.on(",").join(sharedA,sharedB));
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
+ localA.toString());
+
+ try {
+ FSNamesystem.getNamespaceEditsDirs(conf);
+ fail("Allowed multiple shared edits directories");
+ } catch (IOException ioe) {
+ assertEquals("Multiple shared edits directories are not yet supported",
+ ioe.getMessage());
+ }
+ }
/**
* Make sure that the shared edits dirs are listed before non-shared dirs
@@ -78,13 +101,12 @@ public class TestFailureOfSharedDir {
public void testSharedDirsComeFirstInEditsList() throws Exception {
Configuration conf = new Configuration();
URI sharedA = new URI("file:///shared-A");
- URI sharedB = new URI("file:///shared-B");
URI localA = new URI("file:///local-A");
URI localB = new URI("file:///local-B");
URI localC = new URI("file:///local-C");
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
- Joiner.on(",").join(sharedA,sharedB));
+ sharedA.toString());
// List them in reverse order, to make sure they show up in
// the order listed, regardless of lexical sort order.
conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
@@ -93,7 +115,7 @@ public class TestFailureOfSharedDir {
assertEquals(
"Shared dirs should come first, then local dirs, in the order " +
"they were listed in the configuration.",
- Joiner.on(",").join(sharedA, sharedB, localC, localB, localA),
+ Joiner.on(",").join(sharedA, localC, localB, localA),
Joiner.on(",").join(dirs));
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java?rev=1240916&r1=1240915&r2=1240916&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java Mon Feb 6 08:24:11 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.na
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.*;
+import java.io.IOException;
import java.net.URI;
import java.util.Collection;
@@ -87,7 +88,7 @@ public class TestHAConfiguration {
* duplicates removed
*/
@Test
- public void testHAUniqueEditDirs() {
+ public void testHAUniqueEditDirs() throws IOException {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "