You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/07/28 02:19:03 UTC
[hadoop] branch trunk updated: HDFS-16111. Add a configuration to
RoundRobinVolumeChoosingPolicy to avoid failed volumes at datanodes.
(#3175)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ae20516 HDFS-16111. Add a configuration to RoundRobinVolumeChoosingPolicy to avoid failed volumes at datanodes. (#3175)
ae20516 is described below
commit ae20516ebc0709bbe674c4846461ef6d81cc98b4
Author: zhihaixu2012 <zh...@gmail.com>
AuthorDate: Tue Jul 27 19:18:44 2021 -0700
HDFS-16111. Add a configuration to RoundRobinVolumeChoosingPolicy to avoid failed volumes at datanodes. (#3175)
Change-Id: Iead25812d4073e3980893e3e76f7d2b03b57442a
Co-authored-by: Zhihai Xu <zx...@apache.org>
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +++
.../fsdataset/RoundRobinVolumeChoosingPolicy.java | 29 ++++++++++++-
.../src/main/resources/hdfs-default.xml | 11 +++++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 ++
.../TestRoundRobinVolumeChoosingPolicy.java | 48 +++++++++++++++++++++-
5 files changed, 94 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 66e3c8b..808ecfb 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1112,6 +1112,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_THRESHOLD_DEFAULT = 1024L * 1024L * 1024L * 10L; // 10 GB
public static final String DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction";
public static final float DFS_DATANODE_AVAILABLE_SPACE_VOLUME_CHOOSING_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.75f;
+ public static final String
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY =
+ "dfs.datanode.round-robin-volume-choosing-policy.additional-available-space";
+ public static final long
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT =
+ 1024L * 1024L * 1024L; // 1 GB
public static final String DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY =
HdfsClientConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
public static final String DFS_DATANODE_STARTUP_KEY = "dfs.datanode.startup";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
index 0f23e47..fe010b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RoundRobinVolumeChoosingPolicy.java
@@ -17,11 +17,16 @@
*/
package org.apache.hadoop.hdfs.server.datanode.fsdataset;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY;
+
import java.io.IOException;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
@@ -30,7 +35,7 @@ import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
* Use fine-grained locks to synchronize volume choosing.
*/
public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
- implements VolumeChoosingPolicy<V> {
+ implements VolumeChoosingPolicy<V>, Configurable {
public static final Logger LOG =
LoggerFactory.getLogger(RoundRobinVolumeChoosingPolicy.class);
@@ -41,6 +46,9 @@ public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
// syncLocks stores the locks for each storage type.
private Object[] syncLocks;
+ // The required additional available space when choosing a volume.
+ private long additionalAvailableSpace;
+
public RoundRobinVolumeChoosingPolicy() {
int numStorageTypes = StorageType.values().length;
curVolumes = new int[numStorageTypes];
@@ -51,6 +59,23 @@ public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
}
@Override
+ public void setConf(Configuration conf) {
+ additionalAvailableSpace = conf.getLong(
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY,
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_DEFAULT);
+
+ LOG.info("Round robin volume choosing policy initialized: " +
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY +
+ " = " + additionalAvailableSpace);
+ }
+
+ @Override
+ public Configuration getConf() {
+ // Nothing to do. Only added to fulfill the Configurable contract.
+ return null;
+ }
+
+ @Override
public V chooseVolume(final List<V> volumes, long blockSize, String storageId)
throws IOException {
@@ -83,7 +108,7 @@ public class RoundRobinVolumeChoosingPolicy<V extends FsVolumeSpi>
final V volume = volumes.get(curVolume);
curVolume = (curVolume + 1) % volumes.size();
long availableVolumeSize = volume.getAvailable();
- if (availableVolumeSize > blockSize) {
+ if (availableVolumeSize > blockSize + additionalAvailableSpace) {
curVolumes[curVolumeIndex] = curVolume;
return volume;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 58e709f..5818ae1 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2658,6 +2658,17 @@
</property>
<property>
+ <name>dfs.datanode.round-robin-volume-choosing-policy.additional-available-space</name>
+ <value>1073741824</value> <!-- 1 GB -->
+ <description>
+ Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to
+ org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy.
+ This setting controls how much additional available space (unit is byte) is needed
+ when choosing a volume.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.edits.noeditlogchannelflush</name>
<value>false</value>
<description>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index f1b63ac..4de55d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -533,6 +533,9 @@ public class MiniDFSCluster implements AutoCloseable {
DEFAULT_DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD);
this.storagesPerDatanode =
FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs();
+ conf.setLong(DFSConfigKeys
+ .DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY,
+ 0);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
index 44e2a30..fc99d3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/TestRoundRobinVolumeChoosingPolicy.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.datanode.fsdataset;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
import org.apache.hadoop.util.ReflectionUtils;
@@ -70,7 +73,50 @@ public class TestRoundRobinVolumeChoosingPolicy {
// Passed.
}
}
-
+
+ // Test the Round-Robin block-volume choosing algorithm with
+ // additional available space configured.
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRRWithAdditionalAvailableSpace() throws Exception {
+ Configuration conf = new Configuration();
+ // Set the additional available space needed
+ conf.setLong(
+ DFS_DATANODE_ROUND_ROBIN_VOLUME_CHOOSING_POLICY_ADDITIONAL_AVAILABLE_SPACE_KEY,
+ 100);
+ final RoundRobinVolumeChoosingPolicy<FsVolumeSpi> policy =
+ ReflectionUtils.newInstance(RoundRobinVolumeChoosingPolicy.class, conf);
+ testRRWithAdditionalAvailableSpace(policy);
+ }
+
+ public static void testRRWithAdditionalAvailableSpace(
+ VolumeChoosingPolicy<FsVolumeSpi> policy) throws Exception {
+ final List<FsVolumeSpi> volumes = new ArrayList<FsVolumeSpi>();
+
+ // First volume, with 100 bytes of space.
+ volumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(volumes.get(0).getAvailable()).thenReturn(100L);
+
+ // Second volume, with 200 bytes of space.
+ volumes.add(Mockito.mock(FsVolumeSpi.class));
+ Mockito.when(volumes.get(1).getAvailable()).thenReturn(200L);
+
+ // The first volume has only 100L space, so the policy should choose
+ // the second one with additional available space configured as 100L.
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0,
+ null));
+ Assert.assertEquals(volumes.get(1), policy.chooseVolume(volumes, 0,
+ null));
+
+ // Fail if no volume can be chosen?
+ try {
+ policy.chooseVolume(volumes, 100, null);
+ Assert.fail();
+ } catch (IOException e) {
+ // Passed.
+ }
+ }
+
// ChooseVolume should throw DiskOutOfSpaceException
// with volume and block sizes in exception message.
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org