You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jb...@apache.org on 2021/06/25 18:08:54 UTC
[hadoop] branch branch-2.10 updated: HDFS-15659. MiniDFSCluster
dfs.namenode.redundancy.considerLoad default to false (#2443). Contributed
by Ahmed Hussein.
This is an automated email from the ASF dual-hosted git repository.
jbrennan pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 7d231f4 HDFS-15659. MiniDFSCluster dfs.namenode.redundancy.considerLoad default to false (#2443). Contributed by Ahmed Hussein.
7d231f4 is described below
commit 7d231f4b98d539ed8863aca5bf7b686b8ff5160c
Author: Jim Brennan <jb...@apache.org>
AuthorDate: Fri Jun 25 18:06:37 2021 +0000
HDFS-15659. MiniDFSCluster dfs.namenode.redundancy.considerLoad default to false (#2443). Contributed by Ahmed Hussein.
---
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 ++++++++++++++++++++++
.../apache/hadoop/hdfs/TestEncryptedTransfer.java | 5 ++---
.../org/apache/hadoop/hdfs/TestReplication.java | 1 -
3 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index d313e0e..b5b9d04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.NET_TOPOLOGY_NO
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
@@ -168,6 +169,10 @@ public class MiniDFSCluster implements AutoCloseable {
// Changing this default may break some tests that assume it is 2.
private static final int DEFAULT_STORAGES_PER_DATANODE = 2;
+ // do not consider load factor when selecting a datanode.
+ private static final boolean DEFAULT_DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD =
+ false;
+
static { DefaultMetricsSystem.setMiniClusterMode(true); }
public int getStoragesPerDatanode() {
@@ -478,6 +483,19 @@ public class MiniDFSCluster implements AutoCloseable {
}
/**
+ * set the value of DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY in the config
+ * file.
+ *
+ * @param val passed to the flag. This allows overriding the default value
+ * {@link #DEFAULT_DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD}.
+ * @return the builder object.
+ */
+ public Builder setNNRedundancyConsiderLoad(final boolean val) {
+ conf.setBoolean(DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, val);
+ return this;
+ }
+
+ /**
* Construct the actual MiniDFSCluster
*/
public MiniDFSCluster build() throws IOException {
@@ -493,6 +511,10 @@ public class MiniDFSCluster implements AutoCloseable {
DEFAULT_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC);
conf.setLong(DFS_BLOCK_SCANNER_VOLUME_JOIN_TIMEOUT_MSEC_KEY,
defaultScannerVolumeTimeOut);
+ // default is false. do not consider load factor when selecting a
+ // datanode.
+ conf.setBoolean(DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ DEFAULT_DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD);
this.storagesPerDatanode =
FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
index e9a7c42..e32efc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java
@@ -328,9 +328,8 @@ public class TestEncryptedTransfer {
// use 4 datanodes to make sure that after 1 data node is stopped,
// client only retries establishing pipeline with the 4th node.
int numDataNodes = 4;
- // do not consider load factor when selecting a data node
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
- false);
+ // do not consider load factor when selecting a data node.
+ // It is already set to false in the miniCluster
setEncryptionConfigKeys();
cluster = new MiniDFSCluster.Builder(conf)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index daa0ad0..21455b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -243,7 +243,6 @@ public class TestReplication {
*/
public void runReplication(boolean simulated) throws IOException {
Configuration conf = new HdfsConfiguration();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
if (simulated) {
SimulatedFSDataset.setFactory(conf);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org