You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sz...@apache.org on 2022/10/28 06:27:43 UTC
[ozone] branch HDDS-4454 updated: HDDS-7431. [Ozone-Streaming] Disable data steam by default. (#3900)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-4454 by this push:
new 88e1d1ebb2 HDDS-7431. [Ozone-Streaming] Disable data steam by default. (#3900)
88e1d1ebb2 is described below
commit 88e1d1ebb298e45a4b5caa606a12815bfa21bbb9
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Fri Oct 28 14:27:38 2022 +0800
HDDS-7431. [Ozone-Streaming] Disable data steam by default. (#3900)
---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 15 +++++++-------
.../common/src/main/resources/ozone-default.xml | 11 +++++-----
.../transport/server/ratis/XceiverServerRatis.java | 4 ++--
.../container/common/TestDatanodeStateMachine.java | 2 ++
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 ++
.../ozone/container/common/TestEndPoint.java | 4 ++++
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 1 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 ++
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 ++
.../server/TestSecureContainerServer.java | 2 ++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 ++--
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 ++--
.../hadoop/ozone/client/ClientProtocolStub.java | 24 ++++++++++++++++++++++
13 files changed, 59 insertions(+), 18 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 1528c2d854..43281e1a89 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -88,10 +88,10 @@ public final class OzoneConfigKeys {
/**
* Ratis Port where containers listen to datastream requests.
*/
- public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLE
- = "dfs.container.ratis.datastream.enable";
- public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT
- = true;
+ public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLED
+ = "dfs.container.ratis.datastream.enabled";
+ public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT
+ = false;
public static final String DFS_CONTAINER_RATIS_DATASTREAM_PORT
= "dfs.container.ratis.datastream.port";
public static final int DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT
@@ -100,9 +100,10 @@ public final class OzoneConfigKeys {
/**
* Flag to enable ratis streaming on filesystem writes.
*/
- public static final String OZONE_FS_DATASTREAM_ENABLE =
- "ozone.fs.datastream.enable";
- public static final boolean OZONE_FS_DATASTREAM_ENABLE_DEFAULT = false;
+ public static final String OZONE_FS_DATASTREAM_ENABLED
+ = "ozone.fs.datastream.enabled";
+ public static final boolean OZONE_FS_DATASTREAM_ENABLED_DEFAULT
+ = false;
/**
* When set to true, allocate a random free port for ozone container, so that
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 69eed9cd0f..d88e3adc87 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,11 +53,12 @@
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>The ipc port number of container.</description>
</property>
+
<property>
- <name>dfs.container.ratis.datastream.enable</name>
- <value>true</value>
+ <name>dfs.container.ratis.datastream.enabled</name>
+ <value>false</value>
<tag>OZONE, CONTAINER, RATIS, DATASTREAM</tag>
- <description>If enable datastream ipc of container.</description>
+ <description>It specifies whether to enable data stream of container.</description>
</property>
<property>
<name>dfs.container.ratis.datastream.port</name>
@@ -3377,8 +3378,8 @@
The time to live for container location cache in Ozone.
</description>
</property>
-
- <name>ozone.fs.datastream.enable</name>
+ <property>
+ <name>ozone.fs.datastream.enabled</name>
<value>false</value>
<tag>OZONE, DATANODE</tag>
<description>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 6b0ad0e41e..89e5047b85 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -161,8 +161,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
datanodeDetails = dd;
assignPorts();
this.streamEnable = conf.getBoolean(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT);
RaftProperties serverProperties = newRaftProperties();
this.context = context;
this.dispatcher = dispatcher;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 8aa1222233..f2faeaa3a3 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -81,6 +81,8 @@ public class TestDatanodeStateMachine {
TimeUnit.MILLISECONDS);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true);
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
serverAddresses = new ArrayList<>();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
index c004b38843..7af39e5e7f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java
@@ -105,6 +105,8 @@ public class TestDatanodeUpgradeToSchemaV3 {
conf = new OzoneConfiguration();
conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED,
this.schemaV3Enabled);
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 56a04de02c..b620ace2a9 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -144,6 +144,8 @@ public class TestEndPoint {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, 1000)) {
DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
OzoneContainer ozoneContainer = new OzoneContainer(
@@ -170,6 +172,8 @@ public class TestEndPoint {
true);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
true);
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
conf.setFromObject(new ReplicationConfig().setPort(0));
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 4c485d805c..e7a4cf0319 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -321,6 +321,7 @@ public interface MiniOzoneCluster {
protected Boolean randomContainerPort = true;
protected Boolean randomContainerStreamPort = true;
+ protected Boolean enableContainerDatastream = true;
protected Optional<String> datanodeReservedSpace = Optional.empty();
protected Optional<Integer> chunkSize = Optional.empty();
protected OptionalInt streamBufferSize = OptionalInt.empty();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 39f3552b36..d0c897b488 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -933,6 +933,8 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
randomContainerPort);
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
randomContainerPort);
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ enableContainerDatastream);
conf.setBoolean(DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT,
randomContainerStreamPort);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index 7da2bc8e17..4d4e9fb8b6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -216,6 +216,8 @@ public class TestMiniOzoneCluster {
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true);
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
true);
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
ozoneConf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
List<DatanodeStateMachine> stateMachines = new ArrayList<>();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index 51d617a4e0..62e7ecbda6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -216,6 +216,8 @@ public class TestSecureContainerServer {
DatanodeDetails dn, OzoneConfiguration conf) throws IOException {
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
dn.getPort(DatanodeDetails.Port.Name.RATIS).getValue());
+ conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
+ true);
conf.setBoolean(
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true);
final String dir = TEST_DIR + dn.getUuid();
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 8eb92255b6..595381cf49 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -261,8 +261,8 @@ public class BasicOzoneFileSystem extends FileSystem {
private FSDataOutputStream createOutputStream(String key, short replication,
boolean overwrite, boolean recursive) throws IOException {
boolean isRatisStreamingEnabled = getConf().getBoolean(
- OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
- OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED,
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED_DEFAULT);
if (isRatisStreamingEnabled) {
return new FSDataOutputStream(adapter.createStreamFile(key,
replication, overwrite, recursive), statistics);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 75c1da0ec8..c22ff947ba 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -241,8 +241,8 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
private FSDataOutputStream createOutputStream(String key, short replication,
boolean overwrite, boolean recursive) throws IOException {
boolean isRatisStreamingEnabled = getConf().getBoolean(
- OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE,
- OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT);
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED,
+ OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED_DEFAULT);
if (isRatisStreamingEnabled) {
return new FSDataOutputStream(adapter.createStreamFile(key,
replication, overwrite, recursive), statistics);
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index f023488986..38ddef2cb7 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
@@ -570,4 +571,27 @@ public class ClientProtocolStub implements ClientProtocol {
String volumeName, String bucketName, String keyName) throws IOException {
return null;
}
+
+ @Override
+ public OzoneDataStreamOutput createStreamKey(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public OzoneDataStreamOutput createMultipartStreamKey(
+ String volumeName, String bucketName, String keyName, long size,
+ int partNumber, String uploadID) throws IOException {
+ return null;
+ }
+
+ @Override
+ public OzoneDataStreamOutput createStreamFile(
+ String volumeName, String bucketName, String keyName, long size,
+ ReplicationConfig replicationConf, boolean overWrite, boolean recursive)
+ throws IOException {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org