You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2019/10/08 08:57:43 UTC
[hadoop] branch branch-3.1 updated: HDFS-14859. Prevent unnecessary
evaluation of costly operation getNumLiveDataNodes when
dfs.namenode.safemode.min.datanodes is not zero. Contributed by Srinivasu
Majeti.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a3599c7 HDFS-14859. Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero. Contributed by Srinivasu Majeti.
a3599c7 is described below
commit a3599c7e7055e3cf89c75e5c7453bb64f9e98589
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Tue Oct 8 14:14:14 2019 +0530
HDFS-14859. Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero. Contributed by Srinivasu Majeti.
---
.../blockmanagement/BlockManagerSafeMode.java | 16 ++--
.../blockmanagement/TestBlockManagerSafeMode.java | 86 ++++++++++++++++++++++
2 files changed, 97 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index 103f278..883c352 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -567,12 +567,18 @@ class BlockManagerSafeMode {
assert namesystem.hasWriteLock();
// Calculating the number of live datanodes is time-consuming
// in large clusters. Skip it when datanodeThreshold is zero.
- int datanodeNum = 0;
- if (datanodeThreshold > 0) {
- datanodeNum = blockManager.getDatanodeManager().getNumLiveDataNodes();
- }
+ // We need to evaluate getNumLiveDataNodes only when
+ // (blockSafe >= blockThreshold) is true and hence moving evaluation
+ // of datanodeNum conditional to isBlockThresholdMet as well
synchronized (this) {
- return blockSafe >= blockThreshold && datanodeNum >= datanodeThreshold;
+ boolean isBlockThresholdMet = (blockSafe >= blockThreshold);
+ boolean isDatanodeThresholdMet = true;
+ if (isBlockThresholdMet && datanodeThreshold > 0) {
+ int datanodeNum = blockManager.getDatanodeManager().
+ getNumLiveDataNodes();
+ isDatanodeThresholdMet = (datanodeNum >= datanodeThreshold);
+ }
+ return isBlockThresholdMet && isDatanodeThresholdMet;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index 1a6804d..af5342a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -366,6 +366,87 @@ public class TestBlockManagerSafeMode {
}
/**
+ * Test block manager won't leave safe mode if datanode threshold is not met
+ * only if datanodeThreshold is configured > 0.
+ */
+ @Test(timeout = 30000)
+ public void testDatanodeThreshodShouldBeMetOnlyIfConfigured()
+ throws Exception {
+ bmSafeMode.activate(BLOCK_TOTAL);
+
+ //Blocks received is set to threshold
+ setBlockSafe(BLOCK_THRESHOLD);
+
+ //datanodeThreshold is configured to 1 but not all DNs registered .
+ // Expecting safe mode .
+ when(dn.getNumLiveDataNodes()).thenReturn(1);
+ setDatanodeThreshold(1);
+ bmSafeMode.checkSafeMode();
+ assertTrue(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 1 and all DNs registered .
+ // Not expecting safe mode .
+ when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
+ setDatanodeThreshold(1);
+ bmSafeMode.checkSafeMode();
+ waitForExtensionPeriod();
+ assertFalse(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 0 but not all DNs registered .
+ // Not Expecting safe mode .
+ bmSafeMode.activate(BLOCK_TOTAL);
+ setBlockSafe(BLOCK_THRESHOLD);
+ when(dn.getNumLiveDataNodes()).thenReturn(1);
+ setDatanodeThreshold(0);
+ bmSafeMode.checkSafeMode();
+ assertFalse(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 0 and all DNs registered .
+ // Not Expecting safe mode .
+ bmSafeMode.activate(BLOCK_TOTAL);
+ setBlockSafe(BLOCK_THRESHOLD);
+ when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
+ setDatanodeThreshold(0);
+ bmSafeMode.checkSafeMode();
+ assertFalse(bmSafeMode.isInSafeMode());
+
+ //Blocks received set to below threshold and all combinations
+ //of datanodeThreshold should result in safe mode.
+
+
+ //datanodeThreshold is configured to 1 but not all DNs registered .
+ // Expecting safe mode .
+ bmSafeMode.activate(BLOCK_TOTAL);
+ setBlockSafe(BLOCK_THRESHOLD-1);
+ setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
+ when(dn.getNumLiveDataNodes()).thenReturn(1);
+ setDatanodeThreshold(1);
+ bmSafeMode.checkSafeMode();
+ assertTrue(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 1 and all DNs registered .
+ // Expecting safe mode .
+ when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
+ setDatanodeThreshold(1);
+ bmSafeMode.checkSafeMode();
+ assertTrue(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 0 but not all DNs registered .
+ // Expecting safe mode .
+ when(dn.getNumLiveDataNodes()).thenReturn(1);
+ setDatanodeThreshold(0);
+ bmSafeMode.checkSafeMode();
+ assertTrue(bmSafeMode.isInSafeMode());
+
+ //datanodeThreshold is configured to 0 and all DNs registered .
+ // Expecting safe mode .
+ when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
+ setDatanodeThreshold(0);
+ bmSafeMode.checkSafeMode();
+ assertTrue(bmSafeMode.isInSafeMode());
+ }
+
+ /**
* Test block manager won't leave safe mode if there are blocks with
* generation stamp (GS) in future.
*/
@@ -575,6 +656,11 @@ public class TestBlockManagerSafeMode {
Whitebox.setInternalState(bmSafeMode, "blockSafe", blockSafe);
}
+ private void setDatanodeThreshold(int dnSafeModeThreshold) {
+ Whitebox.setInternalState(bmSafeMode, "datanodeThreshold",
+ dnSafeModeThreshold);
+ }
+
private long getblockSafe() {
return (long)Whitebox.getInternalState(bmSafeMode, "blockSafe");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org