You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2026/09/18 18:00:42 UTC
(ozone) branch master updated: HDDS-12688. Set test-friendly defaults in mini-cluster (#10926)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 577f309591b HDDS-12688. Set test-friendly defaults in mini-cluster (#10926)
577f309591b is described below
commit 577f309591b6de98210e55d6cfafd1acfc728281
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Sat Sep 19 02:00:35 2026 +0800
HDDS-12688. Set test-friendly defaults in mini-cluster (#10926)
---
.../hadoop/hdds/conf/OzoneConfiguration.java | 42 ++++++++
.../hadoop/hdds/conf/TestOzoneConfiguration.java | 83 ++++++++++++++++
.../hdds/conf/MutableConfigurationSource.java | 75 +++++++++++++++
.../src/test/resources/ozone-site.xml | 77 +--------------
.../src/test/resources/ozone-site.xml | 86 +----------------
.../hdds/scm/TestXceiverClientManagerSC.java | 4 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 107 +++++++++++++++++++++
...stClientRetryContainerStateMachineFailures.java | 5 +-
.../client/rpc/TestContainerStateMachine.java | 5 +-
.../rpc/TestContainerStateMachineFailures.java | 5 +-
.../client/rpc/TestFailureHandlingByClient.java | 5 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 5 +-
.../org/apache/ozone/test/ClusterForTests.java | 5 +-
.../src/test/resources/ozone-site.xml | 90 +----------------
hadoop-ozone/mini-cluster/pom.xml | 4 +
.../hadoop/ozone/ClientConfigForTesting.java | 54 +++++++++--
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 34 +++++++
17 files changed, 402 insertions(+), 284 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index 69c4c029ce1..c99914f4079 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -34,6 +34,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
@@ -72,6 +73,11 @@ public class OzoneConfiguration extends Configuration implements MutableConfigur
activate();
}
+ /** Ozone's built-in default resources; a value coming only from these may be overridden. */
+ private static final Set<String> DEFAULT_RESOURCES = getConfigurationResourceFiles().stream()
+ .filter(resource -> resource.endsWith("-default.xml"))
+ .collect(Collectors.toSet());
+
private Properties delegatingProps;
public static OzoneConfiguration of(ConfigurationSource source) {
@@ -446,6 +452,42 @@ public synchronized void reloadConfiguration() {
delegatingProps = null;
}
+ /**
+ * Sets {@code value} unless the property was already set explicitly
+ * (programmatically, from the command line, from a {@code *-site.xml}, or from a
+ * user-provided resource). Values that come only from Ozone's built-in default
+ * resources ({@code *-default.xml}) are overridden.
+ * <p>
+ * Hadoop {@link Configuration#setIfUnset(String, String)} uses {@code get(name) == null},
+ * which never succeeds for keys present in default resources after HDDS-12777.
+ */
+ @Override
+ public synchronized void setIfUnset(String name, String value) {
+ if (!isExplicitlySet(name)) {
+ set(name, value);
+ }
+ }
+
+ @Override
+ public synchronized boolean isExplicitlySet(String name) {
+ if (get(name) == null) {
+ return false;
+ }
+ String[] sources = getPropertySources(name);
+ if (sources == null) {
+ return false;
+ }
+ for (String source : sources) {
+ // Explicit unless the value comes only from one of Ozone's built-in default
+ // resources. A user-provided resource is not in that set and is preserved,
+ // even if it happens to be named *-default.xml.
+ if (source != null && !DEFAULT_RESOURCES.contains(source)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
protected final synchronized Properties getProps() {
if (delegatingProps == null) {
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
index 8046deafea6..98f6f87de51 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
@@ -22,9 +22,11 @@
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_HANDLER_COUNT_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -50,6 +52,7 @@
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -301,6 +304,86 @@ public void testInstantiationWithInputConfiguration(@TempDir File tempDir)
assertNotEquals(val, new OzoneConfiguration().get(key));
}
+ @Test
+ public void setIfUnsetOverridesDefaultButKeepsExplicitValue() {
+ final String key = OZONE_SCM_HANDLER_COUNT_KEY;
+ OzoneConfiguration subject = new OzoneConfiguration();
+
+ // Default resources provide a value, so Hadoop's get(key) is non-null.
+ assertNotNull(subject.get(key));
+ String fromDefaults = subject.get(key);
+ MutableConfigurationSource wrapped = MutableConfigurationSource.ifUnsetWrapper(subject);
+ assertThat(wrapped.isExplicitlySet(key)).isFalse();
+
+ subject.setIfUnset(key, "20");
+ assertEquals("20", subject.get(key));
+ assertNotEquals(fromDefaults, subject.get(key));
+ assertThat(wrapped.isExplicitlySet(key)).isTrue();
+
+ subject.set(key, "42");
+ subject.setIfUnset(key, "20");
+ assertEquals("42", subject.get(key));
+ }
+
+ @Test
+ public void setIfUnsetPreservesSiteXmlValue(@TempDir File tempDir)
+ throws IOException {
+ final String key = OZONE_SCM_HANDLER_COUNT_KEY;
+ File ozoneSite = new File(tempDir, "ozone-site.xml");
+ try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
+ Files.newOutputStream(ozoneSite.toPath()), StandardCharsets.UTF_8))) {
+ startConfig(out);
+ appendProperty(out, key, "99");
+ endConfig(out);
+ }
+
+ OzoneConfiguration subject = new OzoneConfiguration();
+ subject.addResource(new Path(ozoneSite.getAbsolutePath()));
+ assertEquals("99", subject.get(key));
+
+ subject.setIfUnset(key, "20");
+ assertEquals("99", subject.get(key));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void setIfUnsetAfterUnset(boolean wrapped) {
+ final String key = OZONE_SCM_HANDLER_COUNT_KEY;
+ conf.set(key, "42");
+ conf.unset(key);
+ assertThat(conf.get(key)).isNull();
+
+ MutableConfigurationSource target = wrapped ? MutableConfigurationSource.ifUnsetWrapper(conf) : conf;
+ assertThat(target.isExplicitlySet(key)).isFalse();
+ target.setIfUnset(key, "20");
+
+ assertThat(conf.get(key)).isEqualTo("20");
+ assertThat(target.isExplicitlySet(key)).isTrue();
+ }
+
+ @Test
+ public void setIfUnsetPreservesCustomResourceValue(@TempDir File tempDir)
+ throws IOException {
+ final String key = OZONE_SCM_HANDLER_COUNT_KEY;
+ // Named *-default.xml on purpose: a user-provided resource is explicit even
+ // when its name matches the built-in default resource convention.
+ File custom = new File(tempDir, "custom-default.xml");
+ try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
+ Files.newOutputStream(custom.toPath()), StandardCharsets.UTF_8))) {
+ startConfig(out);
+ appendProperty(out, key, "77");
+ endConfig(out);
+ }
+
+ OzoneConfiguration subject = new OzoneConfiguration();
+ subject.addResource(new Path(custom.getAbsolutePath()));
+ assertEquals("77", subject.get(key));
+
+ // A value from any non-default resource is explicit and must be preserved.
+ subject.setIfUnset(key, "20");
+ assertEquals("77", subject.get(key));
+ }
+
@Test
public void setConfigFromObjectWithObjectDefaults() {
// GIVEN
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
index 779a8e76417..452edc98557 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/MutableConfigurationSource.java
@@ -17,9 +17,84 @@
package org.apache.hadoop.hdds.conf;
+import java.io.IOException;
+import java.util.Collection;
+
/**
* Configuration that can be both read and written.
*/
public interface MutableConfigurationSource
extends ConfigurationSource, ConfigurationTarget {
+
+ /**
+ * Returns whether a property has an explicitly configured value.
+ * By default, any non-null value is explicit; implementations may distinguish default resources.
+ */
+ default boolean isExplicitlySet(String key) {
+ return get(key) != null;
+ }
+
+ /**
+ * Sets {@code value} for {@code key} only if the key is not already set.
+ * Default implementation treats any non-null {@link #get(String)} result as set.
+ * {@code OzoneConfiguration} (in hdds-common) overrides this to allow
+ * overriding values that come only from default resources.
+ */
+ default void setIfUnset(String key, String value) {
+ if (!isExplicitlySet(key)) {
+ set(key, value);
+ }
+ }
+
+ /**
+ * Creates a wrapper config that changes {@link #set(String, String)} to
+ * {@link #setIfUnset(String, String)}. In other words, value is stored only if
+ * no existing value is explicitly set.
+ */
+ static MutableConfigurationSource ifUnsetWrapper(MutableConfigurationSource wrapped) {
+ return new IfUnsetWrapper(wrapped);
+ }
+
+ /**
+ * Delegates all calls to another configuration object, but changes semantics of
+ * {@link #set(String, String)} to {@link #setIfUnset(String, String)}.
+ */
+ class IfUnsetWrapper implements MutableConfigurationSource {
+
+ private final MutableConfigurationSource wrapped;
+
+ private IfUnsetWrapper(MutableConfigurationSource wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public String get(String key) {
+ return wrapped.get(key);
+ }
+
+ @Override
+ public Collection<String> getConfigKeys() {
+ return wrapped.getConfigKeys();
+ }
+
+ @Override
+ public char[] getPassword(String key) throws IOException {
+ return wrapped.getPassword(key);
+ }
+
+ @Override
+ public void set(String key, String value) {
+ wrapped.setIfUnset(key, value);
+ }
+
+ @Override
+ public void setIfUnset(String key, String value) {
+ wrapped.setIfUnset(key, value);
+ }
+
+ @Override
+ public boolean isExplicitlySet(String key) {
+ return wrapped.isExplicitlySet(key);
+ }
+ }
}
diff --git a/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml
index 95a78dd5f38..c21b140814a 100644
--- a/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml
+++ b/hadoop-ozone/integration-test-recon/src/test/resources/ozone-site.xml
@@ -26,90 +26,15 @@
<name>ozone.om.s3.grpc.server_enabled</name>
<value>false</value>
</property>
- <property>
- <name>hdds.container.ratis.num.write.chunk.threads.per.volume</name>
- <value>4</value>
- </property>
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>20</value>
- </property>
- <property>
- <name>ozone.om.handler.count.key</name>
- <value>20</value>
- </property>
- <property>
- <name>hdds.container.ratis.datastream.enabled</name>
- <value>true</value>
- </property>
- <property>
- <name>hdds.heartbeat.interval</name>
- <value>1s</value>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>100ms</value>
- </property>
<property>
<name>ozone.scm.ratis.pipeline.limit</name>
<value>3</value>
</property>
- <property>
- <name>ozone.scm.close.container.wait.duration</name>
- <value>1s</value>
- </property>
- <property>
- <name>ozone.om.snapshot.diff.job.default.wait.time</name>
- <value>1s</value>
- </property>
- <property>
- <name>hdds.container.ratis.log.appender.queue.byte-limit</name>
- <value>32MB</value>
- </property>
- <property>
- <name>ozone.om.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.scm.ha.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.scm.block.size</name>
- <value>4MB</value>
- </property>
+ <!-- Keep a larger container than MiniOzoneCluster defaults for Recon tests. -->
<property>
<name>ozone.scm.container.size</name>
<value>128MB</value>
</property>
- <property>
- <name>ozone.client.stream.buffer.flush.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.max.size</name>
- <value>2MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.buffer.flush.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.min.packet.size</name>
- <value>256KB</value>
- </property>
- <property>
- <name>ozone.client.datastream.window.size</name>
- <value>8MB</value>
- </property>
<property>
<name>hdds.datanode.volume.min.free.space</name>
<value>5GB</value>
diff --git a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
index da0ea9ab8c3..d400c4ef49c 100644
--- a/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
+++ b/hadoop-ozone/integration-test-s3/src/test/resources/ozone-site.xml
@@ -31,100 +31,16 @@
<value>false</value>
</property>
- <property>
- <name>hdds.container.ratis.num.write.chunk.threads.per.volume</name>
- <value>4</value>
- </property>
-
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>ozone.om.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.datastream.enabled</name>
- <value>true</value>
- </property>
-
-
- <property>
- <name>hdds.heartbeat.interval</name>
- <value>1s</value>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>100ms</value>
- </property>
-
<property>
<name>ozone.scm.ratis.pipeline.limit</name>
<value>3</value>
</property>
- <property>
- <name>ozone.scm.close.container.wait.duration</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>ozone.om.snapshot.diff.job.default.wait.time</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.log.appender.queue.byte-limit</name>
- <value>32MB</value>
- </property>
- <property>
- <name>ozone.om.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.scm.ha.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
-
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.scm.block.size</name>
- <value>4MB</value>
- </property>
+ <!-- Keep a larger container than MiniOzoneCluster defaults for S3 SDK tests. -->
<property>
<name>ozone.scm.container.size</name>
<value>128MB</value>
</property>
- <property>
- <name>ozone.client.stream.buffer.flush.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.max.size</name>
- <value>2MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.buffer.flush.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.min.packet.size</name>
- <value>256KB</value>
- </property>
- <property>
- <name>ozone.client.datastream.window.size</name>
- <value>8MB</value>
- </property>
<property>
<name>hdds.datanode.volume.min.free.space</name>
<value>5GB</value>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
index ff6e49b57ad..88bbfb7969b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestXceiverClientManagerSC.java
@@ -59,9 +59,7 @@ public class TestXceiverClientManagerSC {
@BeforeAll
public static void init() throws Exception {
config = new OzoneConfiguration();
- OzoneClientConfig clientConfig = config.getObject(OzoneClientConfig.class);
- clientConfig.setShortCircuit(true);
- config.setFromObject(clientConfig);
+ config.setBoolean(OzoneClientConfig.OZONE_READ_SHORT_CIRCUIT, true);
config.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, new File(dir, "ozone-socket").getAbsolutePath());
DomainSocket.disableBindPathValidation();
cluster = MiniOzoneCluster.newBuilder(config)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index a227242d302..9f86be490e2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_RANDOM_PORT;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -27,15 +28,20 @@
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageSize;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -49,6 +55,8 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
/**
* Test cases for mini ozone cluster.
@@ -103,6 +111,105 @@ public void testStartMultipleDatanodes() throws Exception {
}
}
+ @Test
+ void testClientDefaultsWithIndividualFlags() {
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.setBoolean("ozone.client.stream.buffer.flush.delay", false);
+ config.setBoolean(OzoneClientConfig.OZONE_READ_SHORT_CIRCUIT, true);
+
+ MiniOzoneCluster.newBuilder(config);
+
+ OzoneClientConfig clientConfig = config.getObject(OzoneClientConfig.class);
+ assertThat(clientConfig.getStreamBufferSize()).isEqualTo(1024 * 1024);
+ assertThat(clientConfig.getStreamBufferFlushSize()).isEqualTo(1024 * 1024);
+ assertThat(clientConfig.getStreamBufferMaxSize()).isEqualTo(2 * 1024 * 1024);
+ assertThat(clientConfig.getDataStreamMinPacketSize()).isEqualTo(256 * 1024);
+ assertThat(clientConfig.getDataStreamBufferFlushSize()).isEqualTo(4 * 1024 * 1024);
+ assertThat(clientConfig.getStreamWindowSize()).isEqualTo(8 * 1024 * 1024);
+ assertThat(clientConfig.isStreamBufferFlushDelay()).isFalse();
+ assertThat(clientConfig.isShortCircuitEnabled()).isTrue();
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "ozone.client.stream.buffer.size, 4MB",
+ "ozone.client.stream.buffer.flush.size, 8MB",
+ "ozone.client.stream.buffer.max.size, 64MB",
+ "ozone.client.stream.buffer.increment, 2MB"
+ })
+ void testClientDefaultsPreserveStreamBuffers(String key, String value) {
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.set(key, value);
+ OzoneClientConfig original = config.getObject(OzoneClientConfig.class);
+
+ MiniOzoneCluster.newBuilder(config);
+ MiniOzoneCluster.newBuilder(config);
+
+ OzoneClientConfig actual = config.getObject(OzoneClientConfig.class);
+ assertThat(actual.getStreamBufferSize()).isEqualTo(original.getStreamBufferSize());
+ assertThat(actual.getStreamBufferFlushSize()).isEqualTo(original.getStreamBufferFlushSize());
+ assertThat(actual.getStreamBufferMaxSize()).isEqualTo(original.getStreamBufferMaxSize());
+ assertThat(config.get(key)).isEqualTo(value);
+ assertThat(actual.getDataStreamBufferFlushSize()).isEqualTo(4 * 1024 * 1024);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "ozone.client.datastream.min.packet.size, 8MB",
+ "ozone.client.datastream.buffer.flush.size, 32MB",
+ "ozone.client.datastream.window.size, 128MB"
+ })
+ void testClientDefaultsPreserveDataStreamBuffers(String key, String value) {
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.set(key, value);
+ OzoneClientConfig original = config.getObject(OzoneClientConfig.class);
+
+ MiniOzoneCluster.newBuilder(config);
+ MiniOzoneCluster.newBuilder(config);
+
+ OzoneClientConfig actual = config.getObject(OzoneClientConfig.class);
+ assertThat(actual.getDataStreamMinPacketSize()).isEqualTo(original.getDataStreamMinPacketSize());
+ assertThat(actual.getDataStreamBufferFlushSize()).isEqualTo(original.getDataStreamBufferFlushSize());
+ assertThat(actual.getStreamWindowSize()).isEqualTo(original.getStreamWindowSize());
+ assertThat(config.get(key)).isEqualTo(value);
+ assertThat(actual.getStreamBufferSize()).isEqualTo(1024 * 1024);
+ }
+
+ @Test
+ void testClientDefaultsCanBeOverridden() {
+ OzoneConfiguration config = new OzoneConfiguration();
+ MiniOzoneCluster.newBuilder(config);
+
+ ClientConfigForTesting.newBuilder(StorageUnit.MB).setChunkSize(2).applyTo(config);
+
+ OzoneClientConfig clientConfig = config.getObject(OzoneClientConfig.class);
+ assertThat(clientConfig.getStreamBufferSize()).isEqualTo(2 * 1024 * 1024);
+ assertThat(clientConfig.getStreamBufferFlushSize()).isEqualTo(2 * 1024 * 1024);
+ assertThat(clientConfig.getStreamBufferMaxSize()).isEqualTo(4 * 1024 * 1024);
+ assertThat(clientConfig.getDataStreamMinPacketSize()).isEqualTo(512 * 1024);
+ assertThat(clientConfig.getDataStreamBufferFlushSize()).isEqualTo(8 * 1024 * 1024);
+ assertThat(clientConfig.getStreamWindowSize()).isEqualTo(16 * 1024 * 1024);
+ }
+
+ @Test
+ void testClientDefaultsPreserveSiteXmlBuffers(@TempDir File testDir) throws IOException {
+ Configuration site = new Configuration(false);
+ site.set("ozone.client.stream.buffer.size", "4MB");
+ File ozoneSite = new File(testDir, "ozone-site.xml");
+ try (OutputStream out = Files.newOutputStream(ozoneSite.toPath())) {
+ site.writeXml(out);
+ }
+ OzoneConfiguration config = new OzoneConfiguration();
+ config.addResource(ozoneSite.toURI().toURL());
+
+ MiniOzoneCluster.newBuilder(config);
+
+ OzoneClientConfig clientConfig = config.getObject(OzoneClientConfig.class);
+ assertThat(clientConfig.getStreamBufferSize()).isEqualTo(4 * 1024 * 1024);
+ assertThat(clientConfig.getStreamBufferFlushSize()).isEqualTo(16 * 1024 * 1024);
+ assertThat(clientConfig.getStreamBufferMaxSize()).isEqualTo(32 * 1024 * 1024);
+ }
+
@Test
void testContainerRandomPort(@TempDir File tempDir) throws IOException {
OzoneConfiguration ozoneConf = SCMTestUtils.getConf(tempDir);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestClientRetryContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestClientRetryContainerStateMachineFailures.java
index 7ba07b5704d..0b2d38e341f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestClientRetryContainerStateMachineFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestClientRetryContainerStateMachineFailures.java
@@ -44,7 +44,6 @@
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -86,9 +85,7 @@ public void init() throws Exception {
conf.setLong(OZONE_SCM_PIPELINE_PER_METADATA_VOLUME, 1);
conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "150s");
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
// update watch timeout to 10 second to finish test for client
RatisClientConfig ratisClientConfig = conf.getObject(RatisClientConfig.class);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
index 9a045178a8c..9790af8217f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
@@ -41,7 +41,6 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl;
import org.apache.hadoop.hdds.utils.IOUtils;
@@ -96,9 +95,7 @@ public void setup() throws Exception {
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s");
conf.set(HddsConfigKeys.HDDS_SCM_SAFEMODE_RULE_REFRESH_INTERVAL, "0s");
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
// conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.toString());
cluster =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
index 0ce38fedc11..0fa6dcfc0ed 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
@@ -67,7 +67,6 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.RatisHelper;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
@@ -132,9 +131,7 @@ public class TestContainerStateMachineFailures {
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index 13f2e458ee6..8b87d62277f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -43,7 +43,6 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -132,9 +131,7 @@ public void init() throws Exception {
raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3));
conf.setFromObject(raftClientConfig);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
conf.setQuietMode(false);
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
index 0feaa33d908..c0db4ecee18 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
@@ -44,7 +44,6 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -84,9 +83,7 @@ public class TestValidateBCSIDOnRestart {
public static void init() throws Exception {
conf = new OzoneConfiguration();
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
index f3823644cde..372eb67b9a7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.jupiter.api.AfterAll;
@@ -61,9 +60,7 @@ protected static OzoneConfiguration createBaseConfiguration() {
raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10));
conf.setFromObject(raftClientConfig);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushDelay(false);
- conf.setFromObject(clientConfig);
+ conf.setBoolean("ozone.client.stream.buffer.flush.delay", false);
conf.setBoolean(OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
diff --git a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
index 2b07b1d060d..f669c4c60c5 100644
--- a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
+++ b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml
@@ -36,97 +36,11 @@
<value>false</value>
</property>
- <property>
- <name>hdds.container.ratis.num.write.chunk.threads.per.volume</name>
- <value>4</value>
- </property>
-
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>ozone.om.handler.count.key</name>
- <value>20</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.datastream.enabled</name>
- <value>true</value>
- </property>
-
-
- <property>
- <name>hdds.heartbeat.interval</name>
- <value>1s</value>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>100ms</value>
- </property>
-
<property>
<name>ozone.scm.ratis.pipeline.limit</name>
<value>3</value>
</property>
- <property>
- <name>ozone.scm.close.container.wait.duration</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>ozone.om.snapshot.diff.job.default.wait.time</name>
- <value>1s</value>
- </property>
-
- <property>
- <name>hdds.container.ratis.log.appender.queue.byte-limit
-</name>
- <value>32MB</value>
- </property>
- <property>
- <name>ozone.om.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.scm.ha.ratis.log.appender.queue.byte-limit</name>
- <value>4MB</value>
- </property>
-
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.scm.block.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.flush.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.max.size</name>
- <value>2MB</value>
- </property>
- <property>
- <name>ozone.client.stream.buffer.size</name>
- <value>1MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.buffer.flush.size</name>
- <value>4MB</value>
- </property>
- <property>
- <name>ozone.client.datastream.min.packet.size</name>
- <value>256KB</value>
- </property>
- <property>
- <name>ozone.client.datastream.window.size</name>
- <value>8MB</value>
- </property>
<property>
<name>ozone.readonly.administrators</name>
<value>admin</value>
@@ -136,8 +50,8 @@
<value>admin</value>
</property>
<property>
- <name>ozone.directory.deleting.service.interval</name>
- <value>2m</value>
+ <name>ozone.directory.deleting.service.interval</name>
+ <value>2m</value>
</property>
</configuration>
diff --git a/hadoop-ozone/mini-cluster/pom.xml b/hadoop-ozone/mini-cluster/pom.xml
index c95b80eeb43..60b234bf9b4 100644
--- a/hadoop-ozone/mini-cluster/pom.xml
+++ b/hadoop-ozone/mini-cluster/pom.xml
@@ -55,6 +55,10 @@
<artifactId>hadoop-common</artifactId>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-client</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
similarity index 67%
rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
rename to hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
index cbf06ec608b..88bb78538c8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
+++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/ClientConfigForTesting.java
@@ -93,6 +93,24 @@ public ClientConfigForTesting setDataStreamWindowSize(long size) {
}
public void applyTo(MutableConfigurationSource conf) {
+ applyTo(conf, false);
+ }
+
+ public void applyTo(MutableConfigurationSource conf, boolean onlyIfUnset) {
+ calculateUndefinedValues();
+
+ final MutableConfigurationSource target =
+ onlyIfUnset ? MutableConfigurationSource.ifUnsetWrapper(conf) : conf;
+ target.setFromObject(getClientConfig(conf, onlyIfUnset));
+
+ if (onlyIfUnset) {
+ setIfUnset(conf);
+ } else {
+ set(conf);
+ }
+ }
+
+ private void calculateUndefinedValues() {
if (streamBufferSize == null) {
streamBufferSize = chunkSize;
}
@@ -114,20 +132,40 @@ public void applyTo(MutableConfigurationSource conf) {
if (blockSize == null) {
blockSize = 2 * streamBufferMaxSize;
}
+ }
+ private OzoneClientConfig getClientConfig(MutableConfigurationSource conf, boolean onlyIfUnset) {
OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferSize(streamBufferSize);
- clientConfig.setStreamBufferMaxSize(streamBufferMaxSize);
- clientConfig.setStreamBufferFlushSize(streamBufferFlushSize);
- clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize);
- clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize);
- clientConfig.setStreamWindowSize(dataStreamWindowSize);
-
- conf.setFromObject(clientConfig);
+ // Preserve related sizes together so partial overrides do not invalidate the caller's configuration.
+ if (!onlyIfUnset || (!conf.isExplicitlySet("ozone.client.stream.buffer.size")
+ && !conf.isExplicitlySet("ozone.client.stream.buffer.flush.size")
+ && !conf.isExplicitlySet("ozone.client.stream.buffer.max.size")
+ && !conf.isExplicitlySet("ozone.client.stream.buffer.increment"))) {
+ clientConfig.setStreamBufferSize(streamBufferSize);
+ clientConfig.setStreamBufferMaxSize(streamBufferMaxSize);
+ clientConfig.setStreamBufferFlushSize(streamBufferFlushSize);
+ }
+ if (!onlyIfUnset || (!conf.isExplicitlySet("ozone.client.datastream.buffer.flush.size")
+ && !conf.isExplicitlySet("ozone.client.datastream.min.packet.size")
+ && !conf.isExplicitlySet("ozone.client.datastream.window.size"))) {
+ clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize);
+ clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize);
+ clientConfig.setStreamWindowSize(dataStreamWindowSize);
+ }
+ return clientConfig;
+ }
+
+ private void set(MutableConfigurationSource conf) {
conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES);
conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize, StorageUnit.BYTES);
}
+ private void setIfUnset(MutableConfigurationSource conf) {
+ final String suffix = StorageUnit.BYTES.getShortName();
+ conf.setIfUnset(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize + suffix);
+ conf.setIfUnset(OZONE_SCM_BLOCK_SIZE, blockSize + suffix);
+ }
+
private long toBytes(long value) {
return Math.round(unit.toBytes(value));
}
diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 8765c2aaaae..fc96311e8f4 100644
--- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -17,6 +17,17 @@
package org.apache.hadoop.ozone;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,6 +37,7 @@
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -272,11 +284,33 @@ abstract class Builder {
protected Builder(OzoneConfiguration conf) {
this.conf = conf;
setClusterId();
+ setDefaultConfigs();
// Use default SCM configurations if no override is provided.
setSCMConfigurator(new SCMConfigurator());
ExitUtils.disableSystemExit();
}
+ /**
+ * Applies test-friendly defaults that reduce memory/disk needs for MiniOzoneCluster.
+ * Explicit values from the caller or from {@code *-site.xml} are preserved.
+ */
+ protected void setDefaultConfigs() {
+ ClientConfigForTesting.newBuilder(StorageUnit.MB)
+ .setChunkSize(1)
+ .applyTo(conf, true);
+
+ conf.setIfUnset(HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, "true");
+ conf.setIfUnset(HDDS_HEARTBEAT_INTERVAL, "1s");
+ conf.setIfUnset(OZONE_OM_SNAPSHOT_DIFF_JOB_DEFAULT_WAIT_TIME, "1s");
+ conf.setIfUnset(OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "1s");
+ conf.setIfUnset(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, "100ms");
+ conf.setIfUnset(OZONE_OM_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT, "4MB");
+ conf.setIfUnset(OZONE_SCM_HA_RAFT_LOG_APPENDER_QUEUE_BYTE_LIMIT, "4MB");
+ conf.setIfUnset(HDDS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME, "4");
+ conf.setIfUnset(OZONE_OM_HANDLER_COUNT_KEY, "20");
+ conf.setIfUnset(OZONE_SCM_HANDLER_COUNT_KEY, "20");
+ }
+
/** Prepare the builder for another call to {@link #build()}, avoiding conflict
* between the clusters created. */
protected void prepareForNextBuild() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org