You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/12/31 02:53:23 UTC
svn commit: r1226020 - in
/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/
Author: todd
Date: Sat Dec 31 01:53:23 2011
New Revision: 1226020
URL: http://svn.apache.org/viewvc?rev=1226020&view=rev
Log:
HDFS-2716. Configuration needs to allow different dfs.http.addresses for each HA NN. Contributed by Todd Lipcon.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt?rev=1226020&r1=1226019&r2=1226020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt Sat Dec 31 01:53:23 2011
@@ -79,3 +79,5 @@ HDFS-1972. Fencing mechanism for block i
HDFS-2714. Fix test cases which use standalone FSNamesystems (todd)
HDFS-2692. Fix bugs related to failover from/into safe mode. (todd)
+
+HDFS-2716. Configuration needs to allow different dfs.http.addresses for each HA NN (todd)
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1226020&r1=1226019&r2=1226020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Sat Dec 31 01:53:23 2011
@@ -337,7 +337,7 @@ public class DFSUtil {
* @param nsId the nameservice ID to look at, or null for non-federated
* @return collection of namenode Ids
*/
- static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
+ public static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
String key = addSuffix(DFS_HA_NAMENODES_KEY, nsId);
return conf.getTrimmedStringCollection(key);
}
@@ -644,24 +644,28 @@ public class DFSUtil {
DFS_NAMENODE_HTTPS_ADDRESS_KEY : DFS_NAMENODE_HTTP_ADDRESS_KEY;
String httpAddressDefault = (securityOn && httpsAddress) ?
DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT : DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
+
+ String suffixes[];
if (namenodeAddr != null) {
// if non-default namenode, try reverse look up
// the nameServiceID if it is available
- String nameServiceId = DFSUtil.getNameServiceIdFromAddress(
- conf, namenodeAddr,
+ suffixes = getSuffixIDs(conf, namenodeAddr,
DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
-
- if (nameServiceId != null) {
- httpAddress = conf.get(DFSUtil.addKeySuffixes(
- httpAddressKey, nameServiceId));
- }
+ } else {
+ suffixes = new String[2];
}
- // else - Use non-federation style configuration
- if (httpAddress == null) {
- httpAddress = conf.get(httpAddressKey, httpAddressDefault);
+
+ return getSuffixedConf(conf, httpAddressKey, httpAddressDefault, suffixes);
+ }
+
+ private static String getSuffixedConf(Configuration conf,
+ String key, String defaultVal, String[] suffixes) {
+ String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes));
+ if (ret != null) {
+ return ret;
}
- return httpAddress;
+ return conf.get(key, defaultVal);
}
/**
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1226020&r1=1226019&r2=1226020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Sat Dec 31 01:53:23 2011
@@ -531,11 +531,12 @@ public class NameNode {
this.conf = conf;
this.role = role;
String nsId = getNameServiceId(conf);
+ String namenodeId = HAUtil.getNameNodeId(conf, nsId);
this.haEnabled = HAUtil.isHAEnabled(conf, nsId);
this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf);
this.haContext = createHAContext();
try {
- initializeGenericKeys(conf, nsId);
+ initializeGenericKeys(conf, nsId, namenodeId);
initialize(conf);
if (!haEnabled) {
state = ACTIVE_STATE;
@@ -852,17 +853,24 @@ public class NameNode {
* @param conf
* Configuration object to lookup specific key and to set the value
* to the key passed. Note the conf object is modified
- * @param nameserviceId name service Id
+ * @param nameserviceId name service Id (to distinguish federated NNs)
+ * @param namenodeId the namenode ID (to distinguish HA NNs)
* @see DFSUtil#setGenericConf(Configuration, String, String, String...)
*/
- public static void initializeGenericKeys(Configuration conf, String
- nameserviceId) {
- String namenodeId = HAUtil.getNameNodeId(conf, nameserviceId);
+ public static void initializeGenericKeys(Configuration conf,
+ String nameserviceId, String namenodeId) {
if ((nameserviceId == null || nameserviceId.isEmpty()) &&
(namenodeId == null || namenodeId.isEmpty())) {
return;
}
+ if (nameserviceId != null) {
+ conf.set(DFS_FEDERATION_NAMESERVICE_ID, nameserviceId);
+ }
+ if (namenodeId != null) {
+ conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId);
+ }
+
DFSUtil.setGenericConf(conf, nameserviceId, namenodeId,
NAMESERVICE_SPECIFIC_KEYS);
if (conf.get(DFS_NAMENODE_RPC_ADDRESS_KEY) != null) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1226020&r1=1226019&r2=1226020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Sat Dec 31 01:53:23 2011
@@ -46,6 +46,7 @@ import static org.apache.hadoop.hdfs.DFS
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
+import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
@@ -177,8 +178,14 @@ public class SecondaryNameNode implement
public SecondaryNameNode(Configuration conf,
CommandLineOpts commandLineOpts) throws IOException {
try {
- NameNode.initializeGenericKeys(conf,
- DFSUtil.getSecondaryNameServiceId(conf));
+ String nsId = DFSUtil.getSecondaryNameServiceId(conf);
+ if (HAUtil.isHAEnabled(conf, nsId)) {
+ LOG.fatal("Cannot use SecondaryNameNode in an HA cluster." +
+ " The Standby Namenode will perform checkpointing.");
+ shutdown();
+ return;
+ }
+ NameNode.initializeGenericKeys(conf, nsId, null);
initialize(conf, commandLineOpts);
} catch(IOException e) {
shutdown();
Modified: hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1226020&r1=1226019&r2=1226020&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Sat Dec 31 01:53:23 2011
@@ -258,20 +258,51 @@ public class TestDFSUtil {
* copied to generic keys when the namenode starts.
*/
@Test
- public void testConfModification() {
+ public void testConfModificationFederationOnly() {
final HdfsConfiguration conf = new HdfsConfiguration();
- conf.set(DFS_FEDERATION_NAMESERVICES, "nn1");
- conf.set(DFS_FEDERATION_NAMESERVICE_ID, "nn1");
- final String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
+ String nsId = "ns1";
+
+ conf.set(DFS_FEDERATION_NAMESERVICES, nsId);
+ conf.set(DFS_FEDERATION_NAMESERVICE_ID, nsId);
+
+ // Set the nameservice specific keys with nameserviceId in the config key
+ for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
+ // Note: value is same as the key
+ conf.set(DFSUtil.addKeySuffixes(key, nsId), key);
+ }
+
+ // Initialize generic keys from specific keys
+ NameNode.initializeGenericKeys(conf, nsId, null);
+
+ // Retrieve the keys without nameserviceId and Ensure generic keys are set
+ // to the correct value
+ for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
+ assertEquals(key, conf.get(key));
+ }
+ }
+
+ /**
+ * Test to ensure nameservice specific keys in the configuration are
+ * copied to generic keys when the namenode starts.
+ */
+ @Test
+ public void testConfModificationFederationAndHa() {
+ final HdfsConfiguration conf = new HdfsConfiguration();
+ String nsId = "ns1";
+ String nnId = "nn1";
+
+ conf.set(DFS_FEDERATION_NAMESERVICES, nsId);
+ conf.set(DFS_FEDERATION_NAMESERVICE_ID, nsId);
+ conf.set(DFS_HA_NAMENODES_KEY + "." + nsId, nnId);
// Set the nameservice specific keys with nameserviceId in the config key
for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
// Note: value is same as the key
- conf.set(DFSUtil.addKeySuffixes(key, nameserviceId), key);
+ conf.set(DFSUtil.addKeySuffixes(key, nsId, nnId), key);
}
// Initialize generic keys from specific keys
- NameNode.initializeGenericKeys(conf, nameserviceId);
+ NameNode.initializeGenericKeys(conf, nsId, nnId);
// Retrieve the keys without nameserviceId and Ensure generic keys are set
// to the correct value