You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by na...@apache.org on 2018/08/03 09:23:15 UTC
hadoop git commit: HDDS-300. Create a config for volume choosing
policy. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/trunk 40ab8ee59 -> db465afb5
HDDS-300. Create a config for volume choosing policy. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db465afb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db465afb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db465afb
Branch: refs/heads/trunk
Commit: db465afb5cf57cce653182b2ae85aa6328f84fdf
Parents: 40ab8ee
Author: Nanda kumar <na...@apache.org>
Authored: Fri Aug 3 14:53:04 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Aug 3 14:53:04 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 4 ++
.../common/src/main/resources/ozone-default.xml | 11 +++++
.../container/keyvalue/KeyValueHandler.java | 14 ++++--
.../container/keyvalue/TestKeyValueHandler.java | 46 +++++++++++++++++++-
4 files changed, 70 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db465afb/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index fd4bf08..d25af80 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -54,4 +54,8 @@ public final class HddsConfigKeys {
public static final int HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT =
20;
+ // Configuration to allow volume choosing policy.
+ public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
+ "hdds.datanode.volume.choosing.policy";
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db465afb/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 6ddf3c6..5099bbe 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -75,6 +75,17 @@
</description>
</property>
<property>
+ <name>hdds.datanode.volume.choosing.policy</name>
+ <value/>
+ <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT</tag>
+ <description>
+ The class name of the policy for choosing volumes in the list of
+ directories. Defaults to
+ org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy.
+ This volume choosing policy selects volumes in a round-robin order.
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.enabled</name>
<value>false</value>
<tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db465afb/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index fac3f3c..a281a53 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -31,8 +31,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .CreateContainerRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.GetSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.KeyValue;
@@ -66,6 +64,7 @@ import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
import org.apache.hadoop.ozone.container.keyvalue.statemachine
.background.BlockDeletingService;
import org.apache.hadoop.util.AutoCloseableLock;
+import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,6 +103,8 @@ import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys
+ .HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
/**
* Handler for KeyValue Container type.
@@ -140,8 +141,9 @@ public class KeyValueHandler extends Handler {
new BlockDeletingService(containerSet, svcInterval, serviceTimeout,
TimeUnit.MILLISECONDS, config);
blockDeletingService.start();
- // TODO: Add supoort for different volumeChoosingPolicies.
- volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
+ volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass(
+ HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
+ .class, VolumeChoosingPolicy.class), conf);
maxContainerSizeGB = config.getInt(ScmConfigKeys
.OZONE_SCM_CONTAINER_SIZE_GB, ScmConfigKeys
.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
@@ -151,6 +153,10 @@ public class KeyValueHandler extends Handler {
openContainerBlockMap = new OpenContainerBlockMap();
}
+ @VisibleForTesting
+ public VolumeChoosingPolicy getVolumeChoosingPolicyForTesting() {
+ return volumeChoosingPolicy;
+ }
/**
* Returns OpenContainerBlockMap instance
* @return OpenContainerBlockMap
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db465afb/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 8e8a1be..747687b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -19,25 +19,36 @@
package org.apache.hadoop.ozone.container.keyvalue;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys
+ .HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.times;
+import java.io.File;
import java.util.UUID;
/**
@@ -193,6 +204,39 @@ public class TestKeyValueHandler {
any(ContainerCommandRequestProto.class), any());
}
+ @Test
+ public void testVolumeSetInKeyValueHandler() throws Exception{
+ File path = GenericTestUtils.getRandomizedTestDir();
+ try {
+ Configuration conf = new OzoneConfiguration();
+ conf.set(HDDS_DATANODE_DIR_KEY, path.getAbsolutePath());
+ ContainerSet cset = new ContainerSet();
+ int[] interval = new int[1];
+ interval[0] = 2;
+ ContainerMetrics metrics = new ContainerMetrics(interval);
+ VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf);
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf, cset,
+ volumeSet, metrics);
+ assertEquals(keyValueHandler.getVolumeChoosingPolicyForTesting()
+ .getClass().getName(), "org.apache.hadoop.ozone.container.common" +
+ ".volume.RoundRobinVolumeChoosingPolicy");
+
+ //Set a class which is not of sub class of VolumeChoosingPolicy
+ conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
+ "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
+ try {
+ new KeyValueHandler(conf, cset, volumeSet, metrics);
+ } catch (RuntimeException ex) {
+ GenericTestUtils.assertExceptionContains("class org.apache.hadoop" +
+ ".ozone.container.common.impl.HddsDispatcher not org.apache" +
+ ".hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy",
+ ex);
+ }
+ } finally {
+ FileUtil.fullyDelete(path);
+ }
+ }
+
private ContainerCommandRequestProto getDummyCommandRequestProto(
ContainerProtos.Type cmdType) {
ContainerCommandRequestProto request =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org