You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2019/07/17 17:37:33 UTC
[hadoop] branch trunk updated: HDDS-1801. Make Topology Aware
Replication/Read non-default for ozone 0.4.1. Contributed by Xiaoyu Yao.
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new eb0849b HDDS-1801. Make Topology Aware Replication/Read non-default for ozone 0.4.1. Contributed by Xiaoyu Yao.
eb0849b is described below
commit eb0849b547bbe753bcd5a84ec6305d933ebe936c
Author: Xiaoyu Yao <xy...@apache.org>
AuthorDate: Mon Jul 15 14:28:53 2019 -0700
HDDS-1801. Make Topology Aware Replication/Read non-default for ozone 0.4.1. Contributed by Xiaoyu Yao.
Signed-off-by: Anu Engineer <ae...@apache.org>
---
.../main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 ++++---
.../src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-----
hadoop-hdds/common/src/main/resources/ozone-default.xml | 6 +++---
.../placement/algorithms/ContainerPlacementPolicyFactory.java | 6 +++++-
hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config | 7 ++++---
5 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 3cc6fa1..77acc42 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -110,7 +110,8 @@ public class XceiverClientGrpc extends XceiverClientSpi {
public void connect() throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
- DatanodeDetails dn = this.pipeline.getClosestNode();
+ DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
+ this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, null);
}
@@ -122,8 +123,8 @@ public class XceiverClientGrpc extends XceiverClientSpi {
public void connect(String encodedToken) throws Exception {
// connect to the closest node, if closest node doesn't exist, delegate to
// first node, which is usually the leader in the pipeline.
- DatanodeDetails dn;
- dn = this.pipeline.getClosestNode();
+ DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
+ this.pipeline.getFirstNode();
// just make a connection to the picked datanode at the beginning
connectToDatanode(dn, encodedToken);
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 4b3b89d..1f194d3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -317,10 +317,6 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
"ozone.scm.container.placement.impl";
- public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
- "org.apache.hadoop.hdds.scm.container.placement.algorithms." +
- "SCMContainerPlacementRackAware";
-
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
@@ -371,7 +367,7 @@ public final class ScmConfigKeys {
public static final String DFS_NETWORK_TOPOLOGY_AWARE_READ_ENABLED =
"dfs.network.topology.aware.read.enable";
public static final String DFS_NETWORK_TOPOLOGY_AWARE_READ_ENABLED_DEFAULT =
- "true";
+ "false";
public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true;
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7fd4ad0..30cf386 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -823,13 +823,13 @@
<property>
<name>ozone.scm.container.placement.impl</name>
<value>
- org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
+ org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom
</value>
<tag>OZONE, MANAGEMENT</tag>
<description>
The full name of class which implements org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy.
The class decides which datanode will be used to host the container replica. If not set,
- org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware will be used as default value.
+ org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default value.
</description>
</property>
<property>
@@ -2347,7 +2347,7 @@
</property>
<property>
<name>dfs.network.topology.aware.read.enable</name>
- <value>true</value>
+ <value>false</value>
<tag>OZONE, PERFORMANCE</tag>
<description>
Whether to enable topology aware read to improve the read performance.
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index 826c3d6..6b1e5e2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -34,6 +34,10 @@ public final class ContainerPlacementPolicyFactory {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerPlacementPolicyFactory.class);
+ private static final Class<? extends ContainerPlacementPolicy>
+ OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
+ SCMContainerPlacementRandom.class;
+
private ContainerPlacementPolicyFactory() {
}
@@ -42,7 +46,7 @@ public final class ContainerPlacementPolicyFactory {
final boolean fallback) throws SCMException{
final Class<? extends ContainerPlacementPolicy> placementClass = conf
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementRackAware.class,
+ OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
ContainerPlacementPolicy.class);
Constructor<? extends ContainerPlacementPolicy> constructor;
try {
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
index f5f8f3f..cfbdfae 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
@@ -26,11 +26,12 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
+OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
+OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
+OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
-HDFS-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
-HDFS-SITE.XML_dfs.network.topology.aware.read.enable=true
ASYNC_PROFILER_HOME=/opt/profiler
LOG4J.PROPERTIES_log4j.rootLogger=DEBUG, ARF
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org