You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2011/02/25 23:30:39 UTC
svn commit: r1074724 - in /hadoop/hdfs/branches/HDFS-1052: ./
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/
Author: jitendra
Date: Fri Feb 25 22:30:38 2011
New Revision: 1074724
URL: http://svn.apache.org/viewvc?rev=1074724&view=rev
Log:
Federation: Multiple namenode configuration.
Added:
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
Modified:
hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Fri Feb 25 22:30:38 2011
@@ -37,6 +37,8 @@ Trunk (unreleased changes)
HDFS-1638.DataNode.handleDiskError needs to inform ALL namenodes if a disk
failed (boryas)
+ HDFS-1647. Federation: Multiple namenode configuration. (jitendra)
+
IMPROVEMENTS
HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java Fri Feb 25 22:30:38 2011
@@ -18,19 +18,32 @@
package org.apache.hadoop.hdfs;
+import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
import java.util.StringTokenizer;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NodeBase;
@InterfaceAudience.Private
public class DFSUtil {
+
+ final private static String PRIMARY_NAMENODE_SUFFIX = "-primary.namenode";
+
/**
* Whether the pathname is valid. Currently prohibits relative paths,
* and names which contain a ":" or "/"
@@ -226,6 +239,72 @@ public class DFSUtil {
return blkLocations;
}
+ private static URI getDefaultNamenode(Configuration conf) {
+ return FileSystem.getDefaultUri(conf);
+ }
+
+ /**
+ * Returns the list of namenode URIs
+ *
+ * @param conf
+ * @return list of namenode URIs
+ */
+ public static List<URI> getNamenodeList(Configuration conf) {
+ Collection<String> namenodes = conf
+ .getStringCollection(DFSConfigKeys.DFS_FEDERATION_NAMENODES);
+
+ List<URI> namenodeUris = new ArrayList<URI>();
+ if (namenodes.isEmpty()) {
+ namenodeUris.add(getDefaultNamenode(conf));
+ } else {
+ Iterator<String> nIter = namenodes.iterator();
+ while (nIter.hasNext()) {
+ namenodeUris.add(URI.create(nIter.next()));
+ }
+ }
+ return namenodeUris;
+ }
+ /**
+ * Returns the hostname of the primary namenode for a given secondary
+ * namenode.
+ *
+ * @param conf
+ * @param secondaryHostName
+ * hostname of the secondary namenode.
+ * @return primary namenode hostname
+ */
+ public static URI getPrimaryNamenode(Configuration conf,
+ String secondaryHostName) {
+ String key = secondaryHostName + PRIMARY_NAMENODE_SUFFIX;
+ String nn = conf.get(key);
+ if (nn == null) {
+ return getDefaultNamenode(conf);
+ } else {
+ return URI.create(nn);
+ }
+ }
+
+ /**
+ * Returns the InetSocketAddresses for each configured namenode
+ * @param conf
+ * @return Array of InetSocketAddresses
+ * @throws IOException
+ */
+ public static InetSocketAddress[] getNNAddresses(Configuration conf)
+ throws IOException {
+ List<URI> nns = getNamenodeList(conf);
+ if (nns == null) {
+ throw new IOException("Federation namnodes are not configured correctly");
+ }
+
+ InetSocketAddress[] isas = new InetSocketAddress[nns.size()];
+ int i = 0;
+ for (URI u : nns) {
+ isas[i++] = NameNode.getAddress(u);
+ }
+ return isas;
+ }
+
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Feb 25 22:30:38 2011
@@ -64,6 +64,7 @@ import org.apache.hadoop.fs.LocalFileSys
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -1143,7 +1144,7 @@ public class DataNode extends Configured
return nameNodeThreads; // already initialized
// get NNs addresses from the configuration
- InetSocketAddress[] isas = NameNode.getNNAddresses(conf);
+ InetSocketAddress[] isas = DFSUtil.getNNAddresses(conf);
AbstractList<BPOfferService> al = new ArrayList<BPOfferService> (isas.length);
for(InetSocketAddress isa : isas) {
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1074724&r1=1074723&r2=1074724&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Feb 25 22:30:38 2011
@@ -221,51 +221,6 @@ public class NameNode implements Namenod
}
/**
- * TODO:FEDERATION
- * at this moment only support fs.default style enteries.
- * @param conf
- * @return array of namenodes' addresses
- */
- public static InetSocketAddress [] getNNAddresses(Configuration conf)
- throws IOException {
- URI[] nns=getNameNodesURIs(conf);
- if(nns == null) {
- throw new IOException("Federation namnodes are not configured correctly");
- }
-
- InetSocketAddress [] isas = new InetSocketAddress[nns.length];
- int i=0;
- for(URI u : nns) {
- isas[i++] = getAddress(u);
- }
- return isas;
- }
-
- /**
- * TODO:FEDERATION
- * get the list of namenodes from the configuration
- * create URI for each one of them
- * @param conf
- * @return list of URIs of all configured NameNodes
- */
- public static URI [] getNameNodesURIs(Configuration conf) {
- String [] nnURIs = conf.getStrings(DFSConfigKeys.DFS_FEDERATION_NAMENODES);
- if(nnURIs == null) {
- nnURIs = new String[] {conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)};
- }
-
- AbstractList<URI> nns = new ArrayList<URI>(nnURIs.length);
- for(String uri : nnURIs) {
- // name should be prepened with FileSystem.fixName(uri)
- // TBD
- nns.add(URI.create(uri));
- }
-
- URI[] r = new URI[nns.size()];
- return nns.toArray(r);
- }
-
- /**
* Set the configuration property for the service rpc address
* to address
*/
Added: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1074724&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java (added)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUtil.java Fri Feb 25 22:30:38 2011
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+
+import java.net.URI;
+import java.util.Iterator;
+import java.util.List;
+
+import org.junit.Test;
+
+
+public class TestDFSUtil {
+
+ @Test
+ public void testMultipleNamenodes() {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.set(DFSConfigKeys.DFS_FEDERATION_NAMENODES, "nn1,nn2");
+ conf.set("sn1-primary.namenode", "nn1");
+ conf.set("sn2-primary.namenode", "nn2");
+ List<URI> namenodes = DFSUtil.getNamenodeList(conf);
+ Iterator<URI> it = namenodes.iterator();
+ assertEquals("nn1", it.next().toString());
+ assertEquals("nn2", it.next().toString());
+ assertEquals("nn1", DFSUtil.getPrimaryNamenode(conf, "sn1").toString());
+ assertEquals("nn2", DFSUtil.getPrimaryNamenode(conf, "sn2").toString());
+ }
+
+ @Test
+ public void testDefaultNamenode() {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ final String hdfs_default = "hdfs://x.y.z:9999/";
+ conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, hdfs_default);
+ List<URI> namenodes = DFSUtil.getNamenodeList(conf);
+ assertEquals(1, namenodes.size());
+ Iterator<URI> it = namenodes.iterator();
+ assertEquals(hdfs_default, it.next().toString());
+ assertEquals(hdfs_default, DFSUtil.getPrimaryNamenode(conf, "sn1")
+ .toString());
+ }
+}