You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by mi...@apache.org on 2015/10/09 20:11:00 UTC
hadoop git commit: HDFS-9142. Separating Configuration object for
namenode(s) in MiniDFSCluster. (Siqi Li via mingma)
Repository: hadoop
Updated Branches:
refs/heads/trunk c11fc8a1b -> de8efc65a
HDFS-9142. Separating Configuration object for namenode(s) in MiniDFSCluster. (Siqi Li via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de8efc65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de8efc65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de8efc65
Branch: refs/heads/trunk
Commit: de8efc65a455c10ae7280b5982c48f9aca84c9d4
Parents: c11fc8a
Author: Ming Ma <mi...@apache.org>
Authored: Fri Oct 9 11:10:46 2015 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Fri Oct 9 11:10:46 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 88 +++++++++++++-------
.../apache/hadoop/hdfs/TestMiniDFSCluster.java | 47 +++++++++++
3 files changed, 106 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8efc65/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7dbdfa8..ce8d79a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1998,6 +1998,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9137. DeadLock between DataNode#refreshVolumes and
BPOfferService#registrationSucceeded. (Uma Maheswara Rao G via yliu)
+ HDFS-9142. Separating Configuration object for namenode(s) in
+ MiniDFSCluster. (Siqi Li via mingma)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8efc65/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 71a4bd2..2473dde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -40,6 +40,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
@@ -890,6 +891,44 @@ public class MiniDFSCluster {
format, operation, clusterId, nnCounter);
nnCounter += nameservice.getNNs().size();
}
+
+ for (NameNodeInfo nn : namenodes.values()) {
+ Configuration nnConf = nn.conf;
+ for (NameNodeInfo nnInfo : namenodes.values()) {
+ if (nn.equals(nnInfo)) {
+ continue;
+ }
+ copyKeys(conf, nnConf, nnInfo.nameserviceId, nnInfo.nnId);
+ }
+ }
+ }
+
+ private static void copyKeys(Configuration srcConf, Configuration destConf,
+ String nameserviceId, String nnId) {
+ String key = DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
+ nameserviceId, nnId);
+ destConf.set(key, srcConf.get(key));
+
+ key = DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY,
+ nameserviceId, nnId);
+ String val = srcConf.get(key);
+ if (val != null) {
+ destConf.set(key, srcConf.get(key));
+ }
+
+ key = DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+ nameserviceId, nnId);
+ val = srcConf.get(key);
+ if (val != null) {
+ destConf.set(key, srcConf.get(key));
+ }
+
+ key = DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+ nameserviceId, nnId);
+ val = srcConf.get(key);
+ if (val != null) {
+ destConf.set(key, srcConf.get(key));
+ }
}
/**
@@ -972,16 +1011,13 @@ public class MiniDFSCluster {
// create all the namenodes in the namespace
nnIndex = nnCounter;
for (NNConf nn : nameservice.getNNs()) {
- initNameNodeConf(conf, nsId, nsCounter, nn.getNnId(), manageNameDfsDirs,
+ Configuration hdfsConf = new Configuration(conf);
+ initNameNodeConf(hdfsConf, nsId, nsCounter, nn.getNnId(), manageNameDfsDirs,
enableManagedDfsDirsRedundancy, nnIndex++);
- NameNodeInfo info = createNameNode(conf, false, operation,
+ createNameNode(hdfsConf, false, operation,
clusterId, nsId, nn.getNnId());
-
// Record the last namenode uri
- if (info != null && info.conf != null) {
- lastDefaultFileSystem =
- info.conf.get(FS_DEFAULT_NAME_KEY);
- }
+ lastDefaultFileSystem = hdfsConf.get(FS_DEFAULT_NAME_KEY);
}
if (!federation && lastDefaultFileSystem != null) {
// Set the default file system to the actual bind address of NN.
@@ -1198,50 +1234,39 @@ public class MiniDFSCluster {
return args;
}
- private NameNodeInfo createNameNode(Configuration conf, boolean format, StartupOption operation,
+ private void createNameNode(Configuration hdfsConf, boolean format, StartupOption operation,
String clusterId, String nameserviceId, String nnId) throws IOException {
// Format and clean out DataNode directories
if (format) {
- DFSTestUtil.formatNameNode(conf);
+ DFSTestUtil.formatNameNode(hdfsConf);
}
if (operation == StartupOption.UPGRADE){
operation.setClusterId(clusterId);
}
-
- // Start the NameNode after saving the default file system.
- String originalDefaultFs = conf.get(FS_DEFAULT_NAME_KEY);
+
String[] args = createArgs(operation);
- NameNode nn = NameNode.createNameNode(args, conf);
+ NameNode nn = NameNode.createNameNode(args, hdfsConf);
if (operation == StartupOption.RECOVER) {
- return null;
+ return;
}
-
// After the NN has started, set back the bound ports into
// the conf
- conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
+ hdfsConf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
nameserviceId, nnId), nn.getNameNodeAddressHostPortString());
if (nn.getHttpAddress() != null) {
- conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY,
+ hdfsConf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY,
nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpAddress()));
}
if (nn.getHttpsAddress() != null) {
- conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY,
+ hdfsConf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTPS_ADDRESS_KEY,
nameserviceId, nnId), NetUtils.getHostPortString(nn.getHttpsAddress()));
}
-
- DFSUtil.setGenericConf(conf, nameserviceId, nnId,
+ copyKeys(hdfsConf, conf, nameserviceId, nnId);
+ DFSUtil.setGenericConf(hdfsConf, nameserviceId, nnId,
DFS_NAMENODE_HTTP_ADDRESS_KEY);
NameNodeInfo info = new NameNodeInfo(nn, nameserviceId, nnId,
- operation, new Configuration(conf));
+ operation, hdfsConf);
namenodes.put(nameserviceId, info);
-
- // Restore the default fs name
- if (originalDefaultFs == null) {
- conf.set(FS_DEFAULT_NAME_KEY, "");
- } else {
- conf.set(FS_DEFAULT_NAME_KEY, originalDefaultFs);
- }
- return info;
}
/**
@@ -2856,7 +2881,7 @@ public class MiniDFSCluster {
*
* @return newly started namenode
*/
- public NameNode addNameNode(Configuration conf, int namenodePort)
+ public void addNameNode(Configuration conf, int namenodePort)
throws IOException {
if(!federation)
throw new IOException("cannot add namenode to non-federated cluster");
@@ -2875,7 +2900,7 @@ public class MiniDFSCluster {
NameNodeInfo[] infos = this.getNameNodeInfos(nameserviceId);
int nnIndex = infos == null ? 0 : infos.length;
initNameNodeConf(conf, nameserviceId, nameServiceIndex, nnId, true, true, nnIndex);
- NameNodeInfo info = createNameNode(conf, true, null, null, nameserviceId, nnId);
+ createNameNode(conf, true, null, null, nameserviceId, nnId);
// Refresh datanodes with the newly started namenode
for (DataNodeProperties dn : dataNodes) {
@@ -2885,7 +2910,6 @@ public class MiniDFSCluster {
// Wait for new namenode to get registrations from all the datanodes
waitActive(nnIndex);
- return info.nameNode;
}
protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8efc65/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index 3fa852e..ec5895b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
@@ -28,6 +29,7 @@ import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.test.PathUtils;
@@ -182,4 +184,49 @@ public class TestMiniDFSCluster {
MiniDFSCluster.shutdownCluster(cluster);
}
}
+
+ @Test
+ public void testSetUpFederatedCluster() throws Exception {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleHAFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ try {
+ cluster.waitActive();
+ cluster.transitionToActive(1);
+ cluster.transitionToActive(3);
+ assertEquals("standby", cluster.getNamesystem(0).getHAState());
+ assertEquals("active", cluster.getNamesystem(1).getHAState());
+ assertEquals("standby", cluster.getNamesystem(2).getHAState());
+ assertEquals("active", cluster.getNamesystem(3).getHAState());
+
+ String ns0nn0 = conf.get(
+ DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns0", "nn0"));
+ String ns0nn1 = conf.get(
+ DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns0", "nn1"));
+ String ns1nn0 = conf.get(
+ DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn0"));
+ String ns1nn1 = conf.get(
+ DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1"));
+
+ for(NameNodeInfo nnInfo : cluster.getNameNodeInfos()) {
+ assertEquals(ns0nn0, nnInfo.conf.get(
+ DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns0", "nn0")));
+ assertEquals(ns0nn1, nnInfo.conf.get(
+ DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns0", "nn1")));
+ assertEquals(ns1nn0, nnInfo.conf.get(
+ DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn0")));
+ assertEquals(ns1nn1, nnInfo.conf.get(
+ DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
+ }
+ } finally {
+ MiniDFSCluster.shutdownCluster(cluster);
+ }
+ }
}