You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2021/12/03 17:19:47 UTC
[hadoop] branch trunk updated: HDFS-16314. Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable (#3664)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c0f405a HDFS-16314. Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable (#3664)
c0f405a is described below
commit c0f405a46b402b4bdf5b7771f5124790fc866965
Author: huhaiyang <hu...@126.com>
AuthorDate: Sat Dec 4 01:19:36 2021 +0800
HDFS-16314. Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable (#3664)
Reviewed-by: Fei Hui <fe...@gmail.com>
Signed-off-by: Akira Ajisaka <aa...@apache.org>
---
.../hdfs/server/blockmanagement/BlockManager.java | 10 ++++++++
.../blockmanagement/BlockPlacementPolicy.java | 12 ++++++++++
.../BlockPlacementPolicyDefault.java | 12 +++++++++-
.../hadoop/hdfs/server/namenode/NameNode.java | 27 ++++++++++++++++------
.../server/namenode/TestNameNodeReconfigure.java | 22 ++++++++++++++++--
.../org/apache/hadoop/hdfs/tools/TestDFSAdmin.java | 6 +++--
6 files changed, 77 insertions(+), 12 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1d4e7d1..9ec9f9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -5360,4 +5360,14 @@ public class BlockManager implements BlockStatsMXBean {
public StoragePolicySatisfyManager getSPSManager() {
return spsManager;
}
+
+ public void setExcludeSlowNodesEnabled(boolean enable) {
+ placementPolicies.getPolicy(CONTIGUOUS).setExcludeSlowNodesEnabled(enable);
+ placementPolicies.getPolicy(STRIPED).setExcludeSlowNodesEnabled(enable);
+ }
+
+ @VisibleForTesting
+ public boolean getExcludeSlowNodesEnabled(BlockType blockType) {
+ return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled();
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 2a212d6..68b3bcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -261,4 +261,16 @@ public abstract class BlockPlacementPolicy {
}
}
}
+
+ /**
+ * Updates the value used for excludeSlowNodesEnabled, which is set by
+ * {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY}
+ * initially.
+ *
+ * @param enable true, we will filter out slow nodes
+ * when choosing targets for blocks, otherwise false not filter.
+ */
+ public abstract void setExcludeSlowNodesEnabled(boolean enable);
+
+ public abstract boolean getExcludeSlowNodesEnabled();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 9d2bb78..dec98d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -103,7 +103,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
protected double considerLoadFactor;
private boolean preferLocalNode;
private boolean dataNodePeerStatsEnabled;
- private boolean excludeSlowNodesEnabled;
+ private volatile boolean excludeSlowNodesEnabled;
protected NetworkTopology clusterMap;
protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats;
@@ -1359,5 +1359,15 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
void setPreferLocalNode(boolean prefer) {
this.preferLocalNode = prefer;
}
+
+ @Override
+ public void setExcludeSlowNodesEnabled(boolean enable) {
+ this.excludeSlowNodesEnabled = enable;
+ }
+
+ @Override
+ public boolean getExcludeSlowNodesEnabled() {
+ return excludeSlowNodesEnabled;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 578656e..1bc8b11 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -189,6 +189,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAM
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT;
import static org.apache.hadoop.util.ExitUtil.terminate;
import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
@@ -331,7 +333,8 @@ public class NameNode extends ReconfigurableBase implements
DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY,
DFS_IMAGE_PARALLEL_LOAD_KEY,
- DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY));
+ DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY,
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY));
private static final String USAGE = "Usage: hdfs namenode ["
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2200,7 +2203,8 @@ public class NameNode extends ReconfigurableBase implements
return newVal;
} else if (property.equals(DFS_IMAGE_PARALLEL_LOAD_KEY)) {
return reconfigureParallelLoad(newVal);
- } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) {
+ } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)
+ || (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY))) {
return reconfigureSlowNodesParameters(datanodeManager, property, newVal);
} else {
throw new ReconfigurationException(property, newVal, getConf().get(
@@ -2234,7 +2238,7 @@ public class NameNode extends ReconfigurableBase implements
newSetting = bm.getBlocksReplWorkMultiplier();
} else {
throw new IllegalArgumentException("Unexpected property " +
- property + "in reconfReplicationParameters");
+ property + " in reconfReplicationParameters");
}
LOG.info("RECONFIGURE* changed {} to {}", property, newSetting);
return String.valueOf(newSetting);
@@ -2390,15 +2394,24 @@ public class NameNode extends ReconfigurableBase implements
String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager,
final String property, final String newVal) throws ReconfigurationException {
+ BlockManager bm = namesystem.getBlockManager();
namesystem.writeLock();
boolean enable;
try {
- if (newVal == null) {
- enable = DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT;
+ if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) {
+ enable = (newVal == null ? DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT :
+ Boolean.parseBoolean(newVal));
+ datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable);
+ } else if (property.equals(
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) {
+ enable = (newVal == null ?
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT :
+ Boolean.parseBoolean(newVal));
+ bm.setExcludeSlowNodesEnabled(enable);
} else {
- enable = Boolean.parseBoolean(newVal);
+ throw new IllegalArgumentException("Unexpected property " +
+ property + " in reconfigureSlowNodesParameters");
}
- datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable);
LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
return Boolean.toString(enable);
} catch (IllegalArgumentException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index eed442c..da9b447 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -34,10 +34,12 @@ import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode;
+import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfyManager;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
@@ -52,6 +54,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT;
public class TestNameNodeReconfigure {
@@ -399,8 +402,8 @@ public class TestNameNodeReconfigure {
public void testEnableSlowNodesParametersAfterReconfigured()
throws ReconfigurationException {
final NameNode nameNode = cluster.getNameNode();
- final DatanodeManager datanodeManager = nameNode.namesystem
- .getBlockManager().getDatanodeManager();
+ final BlockManager blockManager = nameNode.namesystem.getBlockManager();
+ final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
// By default, avoidSlowDataNodesForRead is false.
assertEquals(false, datanodeManager.getEnableAvoidSlowDataNodesForRead());
@@ -410,6 +413,21 @@ public class TestNameNodeReconfigure {
// After reconfigured, avoidSlowDataNodesForRead is true.
assertEquals(true, datanodeManager.getEnableAvoidSlowDataNodesForRead());
+
+ // By default, excludeSlowNodesEnabled is false.
+ assertEquals(false, blockManager.
+ getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
+ assertEquals(false, blockManager.
+ getExcludeSlowNodesEnabled(BlockType.STRIPED));
+
+ nameNode.reconfigureProperty(
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, Boolean.toString(true));
+
+ // After reconfigured, excludeSlowNodesEnabled is true.
+ assertEquals(true, blockManager.
+ getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
+ assertEquals(true, blockManager.
+ getExcludeSlowNodesEnabled(BlockType.STRIPED));
}
@After
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index 76ba558..6cbcc35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -39,6 +39,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHEC
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
import org.apache.commons.io.FileUtils;
import org.apache.commons.text.TextStringBuilder;
@@ -431,13 +432,14 @@ public class TestDFSAdmin {
final List<String> outs = Lists.newArrayList();
final List<String> errs = Lists.newArrayList();
getReconfigurableProperties("namenode", address, outs, errs);
- assertEquals(14, outs.size());
+ assertEquals(15, outs.size());
assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(1));
assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(2));
assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(3));
assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(4));
assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(5));
- assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(6));
+ assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(6));
+ assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(7));
assertEquals(errs.size(), 0);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org