You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ta...@apache.org on 2021/12/14 08:24:36 UTC
[hadoop] branch branch-3.2 updated: HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new a57e83a HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
a57e83a is described below
commit a57e83a47ba9322ee4a21dd1a66001f7f30851ba
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Tue Dec 14 13:51:51 2021 +0530
HDFS-16373. Fix MiniDFSCluster restart in case of multiple namenodes. (#3756)
Reviewed-by: Viraj Jasani <vj...@apache.org>
Reviewed-by: litao <to...@gmail.com>
Signed-off-by: Takanobu Asanuma <ta...@apache.org>
---
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 43 +++++++++++-----------
.../org/apache/hadoop/hdfs/TestMiniDFSCluster.java | 8 ++++
2 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 1aff916..c5c33b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2234,9 +2234,11 @@ public class MiniDFSCluster implements AutoCloseable {
info.nameNode = nn;
info.setStartOpt(startOpt);
if (waitActive) {
- waitClusterUp();
+ if (numDataNodes > 0) {
+ waitNameNodeUp(nnIndex);
+ }
LOG.info("Restarted the namenode");
- waitActive();
+ waitActive(nnIndex);
}
}
@@ -2746,11 +2748,25 @@ public class MiniDFSCluster implements AutoCloseable {
DFSClient client = new DFSClient(addr, conf);
// ensure all datanodes have registered and sent heartbeat to the namenode
- while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+ int failedCount = 0;
+ while (true) {
try {
- LOG.info("Waiting for cluster to become active");
- Thread.sleep(100);
+ while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+ LOG.info("Waiting for cluster to become active");
+ Thread.sleep(100);
+ }
+ break;
+ } catch (IOException e) {
+ failedCount++;
+ // Cached RPC connection to namenode, if any, is expected to fail once
+ if (failedCount > 1) {
+ LOG.warn("Tried waitActive() " + failedCount
+ + " time(s) and failed, giving up. " + StringUtils
+ .stringifyException(e));
+ throw e;
+ }
} catch (InterruptedException e) {
+ throw new IOException(e);
}
}
@@ -2786,22 +2802,7 @@ public class MiniDFSCluster implements AutoCloseable {
*/
public void waitActive() throws IOException {
for (int index = 0; index < namenodes.size(); index++) {
- int failedCount = 0;
- while (true) {
- try {
- waitActive(index);
- break;
- } catch (IOException e) {
- failedCount++;
- // Cached RPC connection to namenode, if any, is expected to fail once
- if (failedCount > 1) {
- LOG.warn("Tried waitActive() " + failedCount
- + " time(s) and failed, giving up. "
- + StringUtils.stringifyException(e));
- throw e;
- }
- }
- }
+ waitActive(index);
}
LOG.info("Cluster is active");
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index d60e025..6403cc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -309,6 +309,14 @@ public class TestMiniDFSCluster {
DFSUtil.addKeySuffixes(
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
}
+
+ // Shutdown namenodes individually.
+ cluster.shutdownNameNode(0);
+ cluster.shutdownNameNode(1);
+
+ // Restart namenodes individually with wait active, both should be successful.
+ cluster.restartNameNode(0);
+ cluster.restartNameNode(1);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org