You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/07/29 14:00:03 UTC
[hadoop] branch branch-3.1 updated: HDFS-14632. Reduce useless
#getNumLiveDataNodes call in SafeModeMonitor. Contributed by He Xiaoqiao.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2e1afa9 HDFS-14632. Reduce useless #getNumLiveDataNodes call in SafeModeMonitor. Contributed by He Xiaoqiao.
2e1afa9 is described below
commit 2e1afa9a867a61548963b682f0b98c417f858dff
Author: Yiqun Lin <yq...@apache.org>
AuthorDate: Wed Jul 10 10:53:34 2019 +0800
HDFS-14632. Reduce useless #getNumLiveDataNodes call in SafeModeMonitor. Contributed by He Xiaoqiao.
(cherry picked from commit 993dc8726b7d40ac832ae3e23b64e8541b62c4bd)
(cherry picked from commit 69f1c8d77b941adf4a279af4e4511e48eebe8d86)
---
.../blockmanagement/BlockManagerSafeMode.java | 22 +++++++++-------
.../java/org/apache/hadoop/hdfs/TestSafeMode.java | 10 ++++----
.../blockmanagement/TestBlockManagerSafeMode.java | 25 +++++++++++++++++++
.../hdfs/server/namenode/ha/TestHASafeMode.java | 29 ++++++++++++++--------
4 files changed, 62 insertions(+), 24 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index b730f61..103f278 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -309,16 +309,20 @@ class BlockManagerSafeMode {
}
}
- int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes();
- if (numLive < datanodeThreshold) {
- msg += String.format(
- "The number of live datanodes %d needs an additional %d live "
- + "datanodes to reach the minimum number %d.%n",
- numLive, (datanodeThreshold - numLive), datanodeThreshold);
+ if (datanodeThreshold > 0) {
+ int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes();
+ if (numLive < datanodeThreshold) {
+ msg += String.format(
+ "The number of live datanodes %d needs an additional %d live "
+ + "datanodes to reach the minimum number %d.%n",
+ numLive, (datanodeThreshold - numLive), datanodeThreshold);
+ } else {
+ msg += String.format("The number of live datanodes %d has reached "
+ + "the minimum number %d. ",
+ numLive, datanodeThreshold);
+ }
} else {
- msg += String.format("The number of live datanodes %d has reached "
- + "the minimum number %d. ",
- numLive, datanodeThreshold);
+ msg += "The minimum number of live datanodes is not required. ";
}
if (getBytesInFuture() > 0) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index f25d28f..77ef959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -201,11 +201,11 @@ public class TestSafeMode {
final NameNode nn = cluster.getNameNode();
String status = nn.getNamesystem().getSafemode();
- assertEquals("Safe mode is ON. The reported blocks 0 needs additional " +
- "14 blocks to reach the threshold 0.9990 of total blocks 15." + NEWLINE +
- "The number of live datanodes 0 has reached the minimum number 0. " +
- "Safe mode will be turned off automatically once the thresholds " +
- "have been reached.", status);
+ assertEquals("Safe mode is ON. The reported blocks 0 needs additional "
+ + "14 blocks to reach the threshold 0.9990 of total blocks 15."
+ + NEWLINE + "The minimum number of live datanodes is not required. "
+ + "Safe mode will be turned off automatically once the thresholds have "
+ + "been reached.", status);
assertFalse("Mis-replicated block queues should not be initialized " +
"until threshold is crossed",
NameNodeAdapter.safeModeInitializedReplQueues(nn));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index 6f97401..1a6804d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -457,6 +457,31 @@ public class TestBlockManagerSafeMode {
}
/**
+ * Test get safe mode tip without minimum number of live datanodes required.
+ */
+ @Test
+ public void testGetSafeModeTipsWithoutNumLiveDatanode() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.setDouble(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,
+ THRESHOLD);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY,
+ EXTENSION);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 0);
+
+ NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
+
+ BlockManager blockManager = spy(new BlockManager(fsn, false, conf));
+
+ BlockManagerSafeMode safeMode = new BlockManagerSafeMode(blockManager,
+ fsn, false, conf);
+ safeMode.activate(BLOCK_TOTAL);
+ String tip = safeMode.getSafeModeTip();
+
+ assertTrue(tip.contains("The minimum number of live datanodes is not "
+ + "required."));
+ }
+
+ /**
* Test get safe mode tip in case of blocks with future GS.
*/
@Test(timeout = 30000)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index ee8ee90..6e7bd15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
@@ -493,13 +493,22 @@ public class TestHASafeMode {
int numNodes, int nodeThresh) {
String status = nn.getNamesystem().getSafemode();
if (safe == total) {
- assertTrue("Bad safemode status: '" + status + "'",
- status.startsWith(
- "Safe mode is ON. The reported blocks " + safe + " has reached the "
- + "threshold 0.9990 of total blocks " + total + ". The number of "
- + "live datanodes " + numNodes + " has reached the minimum number "
- + nodeThresh + ". In safe mode extension. "
- + "Safe mode will be turned off automatically"));
+ if (nodeThresh == 0) {
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith("Safe mode is ON. The reported blocks " + safe
+ + " has reached the " + "threshold 0.9990 of total blocks "
+ + total + ". The minimum number of live datanodes is not "
+ + "required. In safe mode extension. Safe mode will be turned "
+ + "off automatically"));
+ } else {
+ assertTrue("Bad safemode status: '" + status + "'",
+ status.startsWith(
+ "Safe mode is ON. The reported blocks " + safe + " has reached "
+ + "the threshold 0.9990 of total blocks " + total + ". The "
+ + "number of live datanodes " + numNodes + " has reached "
+ + "the minimum number " + nodeThresh + ". In safe mode "
+ + "extension. Safe mode will be turned off automatically"));
+ }
} else {
int additional = (int) (total * 0.9990) - safe;
assertTrue("Bad safemode status: '" + status + "'",
@@ -568,9 +577,9 @@ public class TestHASafeMode {
assertTrue("Bad safemode status: '" + status + "'",
status.startsWith(
"Safe mode is ON. The reported blocks 10 has reached the threshold "
- + "0.9990 of total blocks 10. The number of live datanodes 3 has "
- + "reached the minimum number 0. In safe mode extension. "
- + "Safe mode will be turned off automatically"));
+ + "0.9990 of total blocks 10. The minimum number of live datanodes is "
+ + "not required. In safe mode extension. Safe mode will be turned off "
+ + "automatically"));
// Delete those blocks while the SBN is in safe mode.
// Immediately roll the edit log before the actual deletions are sent
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org