You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2022/04/12 18:50:06 UTC
[ozone] branch HDDS-3816-ec updated: HDDS-6565. EC: Unify replication-related CLI params (#3293)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
e38607b0cd is described below
commit e38607b0cde1591097726b42cb85913b93a39b43
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Tue Apr 12 20:50:01 2022 +0200
HDDS-6565. EC: Unify replication-related CLI params (#3293)
---
.../hadoop/hdds/client/ReplicationConfig.java | 15 ++
.../hadoop/ozone/freon/TestDataValidate.java | 45 +++---
.../freon/TestFreonWithDatanodeFastRestart.java | 25 ++--
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 ++--
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 152 +++++++++++----------
.../hadoop/ozone/shell/TestOzoneShellHA.java | 14 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 5 +-
.../ozone/freon/FreonReplicationOptions.java | 73 ++++++++++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 ++-
.../ozone/freon/OzoneClientKeyGenerator.java | 38 +-----
.../hadoop/ozone/freon/RandomKeyGenerator.java | 99 ++------------
.../hadoop/ozone/shell/ReplicationOptions.java | 87 ++++++++++++
.../ozone/shell/SetReplicationConfigOptions.java | 44 ------
.../ozone/shell/ShellReplicationOptions.java | 34 +++++
.../ozone/shell/bucket/CreateBucketHandler.java | 41 +-----
.../shell/bucket/SetReplicationConfigHandler.java | 23 ++--
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +--
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 23 +---
18 files changed, 399 insertions(+), 384 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
index 35c5155559..c9a12a64e6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java
@@ -159,6 +159,21 @@ public interface ReplicationConfig {
replication = Objects.toString(replication,
config.get(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT));
+ return parseWithoutFallback(type, replication, config);
+ }
+
+ static ReplicationConfig parseWithoutFallback(ReplicationType type,
+ String replication, ConfigurationSource config) {
+
+ if (replication == null) {
+ throw new IllegalArgumentException(
+ "Replication can't be null. Replication type passed was : " + type);
+ }
+ if (type == null) {
+ throw new IllegalArgumentException(
+ "Replication type must be specified for: " + replication);
+ }
+
ReplicationConfig replicationConfig;
switch (type) {
case RATIS:
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
index d14238417f..b9eb231ff8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ozone.freon;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -27,6 +25,7 @@ import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.Assert;
import org.junit.Test;
+import picocli.CommandLine;
import java.time.Duration;
@@ -64,17 +63,19 @@ public abstract class TestDataValidate {
}
@Test
- public void ratisTestLargeKey() throws Exception {
+ public void ratisTestLargeKey() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setKeySize(20971520);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
+ new RandomKeyGenerator(cluster.getConf());
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--key-size", "20971520",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
@@ -82,16 +83,18 @@ public abstract class TestDataValidate {
}
@Test
- public void validateWriteTest() throws Exception {
+ public void validateWriteTest() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.call();
+ new RandomKeyGenerator(cluster.getConf());
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "2",
+ "--num-of-buckets", "5",
+ "--num-of-keys", "10",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
index 957c3d6fc2..2c11804e99 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeFastRestart.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ozone.freon;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.container.TestHelper;
@@ -34,6 +32,7 @@ import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import picocli.CommandLine;
/**
* Tests Freon with Datanode restarts without waiting for pipeline to close.
@@ -111,17 +110,19 @@ public class TestFreonWithDatanodeFastRestart {
startFreon();
}
- private void startFreon() throws Exception {
+ private void startFreon() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setKeySize(20971520);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
+ new RandomKeyGenerator(cluster.getConf());
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--key-size", "20971520",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
index 290eb5d792..50f49713bf 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithPipelineDestroy.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -36,6 +34,7 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import picocli.CommandLine;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
@@ -102,17 +101,19 @@ public class TestFreonWithPipelineDestroy {
startFreon();
}
- private void startFreon() throws Exception {
+ private void startFreon() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setKeySize(20971520);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--key-size", "20971520",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
index 4b51107ddb..132d086dc0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.ozone.freon;
import java.time.Duration;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
@@ -32,6 +30,7 @@ import org.junit.Assert;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import picocli.CommandLine;
/**
* Tests Freon, with MiniOzoneCluster.
@@ -77,80 +76,89 @@ public class TestRandomKeyGenerator {
}
@Test
- public void testDefault() throws Exception {
+ void testDefault() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "2",
+ "--num-of-buckets", "5",
+ "--num-of-keys", "10");
+
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
- public void testECKey() throws Exception {
+ void testECKey() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setReplication("rs-3-2-1024k");
- randomKeyGenerator.setType(ReplicationType.EC);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "2",
+ "--num-of-buckets", "5",
+ "--num-of-keys", "10",
+ "--replication", "rs-3-2-1024k",
+ "--type", "EC"
+ );
+
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
- public void testMultiThread() throws Exception {
+ void testMultiThread() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(10);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.setKeySize(10240);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "10",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "10",
+ "--num-of-threads", "10",
+ "--key-size", "10240",
+ "--factor", "THREE",
+ "--type", "RATIS"
+ );
+
Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
- public void testRatisKey() throws Exception {
+ void testRatisKey() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(10);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.setKeySize(10240);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "10",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "10",
+ "--num-of-threads", "10",
+ "--key-size", "10240",
+ "--factor", "THREE",
+ "--type", "RATIS"
+ );
+
Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
- public void testKeyLargerThan2GB() throws Exception {
+ void testKeyLargerThan2GB() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setNumOfThreads(1);
- randomKeyGenerator.setKeySize(10L + Integer.MAX_VALUE);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--num-of-threads", "1",
+ "--key-size", String.valueOf(10L + Integer.MAX_VALUE),
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
@@ -158,18 +166,20 @@ public class TestRandomKeyGenerator {
}
@Test
- public void testZeroSizeKey() throws Exception {
+ void testZeroSizeKey() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setNumOfThreads(1);
- randomKeyGenerator.setKeySize(0);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--num-of-threads", "1",
+ "--key-size", "0",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--validate-writes"
+ );
+
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
@@ -177,33 +187,37 @@ public class TestRandomKeyGenerator {
}
@Test
- public void testThreadPoolSize() throws Exception {
+ void testThreadPoolSize() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
+ "--num-of-threads", "10",
+ "--factor", "THREE",
+ "--type", "RATIS"
+ );
+
Assert.assertEquals(10, randomKeyGenerator.getThreadPoolSize());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
@Flaky("HDDS-5993")
- public void cleanObjectsTest() throws Exception {
+ void cleanObjectsTest() {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator(cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.setCleanObjects(true);
- randomKeyGenerator.call();
+ CommandLine cmd = new CommandLine(randomKeyGenerator);
+ cmd.execute("--num-of-volumes", "2",
+ "--num-of-buckets", "5",
+ "--num-of-keys", "10",
+ "--num-of-threads", "10",
+ "--factor", "THREE",
+ "--type", "RATIS",
+ "--clean-objects"
+ );
+
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index dc5c8bbb32..a09af96c15 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -108,8 +108,8 @@ public class TestOzoneShellHA {
private static String testFilePathString;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
- private static OzoneShell ozoneShell = null;
- private static OzoneAdmin ozoneAdminShell = null;
+ private OzoneShell ozoneShell = null;
+ private OzoneAdmin ozoneAdminShell = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final ByteArrayOutputStream err = new ByteArrayOutputStream();
@@ -141,9 +141,6 @@ public class TestOzoneShellHA {
testFile.getParentFile().mkdirs();
testFile.createNewFile();
- ozoneShell = new OzoneShell();
- ozoneAdminShell = new OzoneAdmin();
-
// Init HA cluster
omServiceId = "om-service-test1";
numOfOMs = 3;
@@ -176,6 +173,8 @@ public class TestOzoneShellHA {
@Before
public void setup() throws UnsupportedEncodingException {
+ ozoneShell = new OzoneShell();
+ ozoneAdminShell = new OzoneAdmin();
System.setOut(new PrintStream(out, false, DEFAULT_ENCODING));
System.setErr(new PrintStream(err, false, DEFAULT_ENCODING));
}
@@ -1004,8 +1003,9 @@ public class TestOzoneShellHA {
execute(ozoneShell, args);
Assert.fail("Must throw Exception when missing replication param");
} catch (Exception e) {
- Assert.assertEquals(e.getCause().getMessage(),
- "Replication can't be null. Replication type passed was : EC");
+ Assert.assertEquals(
+ "Replication can't be null. Replication type passed was : EC",
+ e.getCause().getMessage());
}
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
index 1638b1c33f..14021da613 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
@@ -95,7 +94,7 @@ public final class OzoneClientUtils {
*/
public static ReplicationConfig resolveClientSideReplicationConfig(
short replication, ReplicationConfig clientConfiguredReplConfig,
- ReplicationConfig bucketReplConfig, OzoneConfiguration config) {
+ ReplicationConfig bucketReplConfig, ConfigurationSource config) {
ReplicationConfig clientDeterminedReplConfig = null;
boolean isECBucket = bucketReplConfig != null && bucketReplConfig
@@ -165,7 +164,7 @@ public final class OzoneClientUtils {
*/
public static ReplicationConfig validateAndGetClientReplicationConfig(
ReplicationType userPassedType, String userPassedReplication,
- OzoneConfiguration clientSideConfig) {
+ ConfigurationSource clientSideConfig) {
// Priority 1: User passed replication config values.
// Priority 2: Client side configured replication config values.
/* if above two are not available, we should just return null and clients
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
new file mode 100644
index 0000000000..3e5ac589eb
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.shell.ReplicationOptions;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Spec;
+
+import java.util.Optional;
+
+import static picocli.CommandLine.Spec.Target.MIXEE;
+
+/**
+ * Options for specifying replication config for Freon.
+ */
+public class FreonReplicationOptions extends ReplicationOptions {
+
+ private static final String FACTOR_OPT = "--factor";
+
+ private ReplicationFactor factor;
+
+ @Spec(MIXEE)
+ private CommandSpec spec;
+
+ @Option(names = { "-F", FACTOR_OPT },
+ description = "[deprecated] Replication factor (ONE, THREE)",
+ defaultValue = "THREE"
+ )
+ public void setFactor(ReplicationFactor factor) {
+ this.factor = factor;
+ }
+
+ // -t is already taken for number of threads
+ @Option(names = {"--type", "--replication-type"},
+ description = "Replication type. Supported types are: RATIS, EC")
+ @Override
+ public void setType(ReplicationType type) {
+ super.setType(type);
+ }
+
+ /**
+ * Support legacy --factor option.
+ */
+ @Override
+ public Optional<ReplicationConfig> fromParams(ConfigurationSource conf) {
+ if (spec.commandLine().getParseResult().hasMatchedOption(FACTOR_OPT)) {
+ return Optional.of(ReplicationConfig.fromTypeAndFactor(
+ ReplicationType.RATIS, factor));
+ }
+
+ return super.fromParams(conf);
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
index 53a7705f81..28d33282a0 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
@@ -20,9 +20,8 @@ import java.util.ArrayList;
import java.util.concurrent.Callable;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs.Builder;
@@ -33,6 +32,7 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import com.codahale.metrics.Timer;
import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL;
@@ -61,11 +61,8 @@ public class OmKeyGenerator extends BaseFreonGenerator
defaultValue = "bucket1")
private String bucketName;
- @Option(names = { "-F", "--factor" },
- description = "Replication factor (ONE, THREE)",
- defaultValue = "THREE"
- )
- private ReplicationFactor factor = ReplicationFactor.THREE;
+ @Mixin
+ private FreonReplicationOptions replication;
@Option(
names = "--om-service-id",
@@ -76,19 +73,20 @@ public class OmKeyGenerator extends BaseFreonGenerator
private OzoneManagerProtocol ozoneManagerClient;
private Timer timer;
+ private ReplicationConfig replicationConfig;
@Override
public Void call() throws Exception {
init();
- OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+ OzoneConfiguration conf = createOzoneConfiguration();
+ replicationConfig = replication.fromParams(conf).orElse(null);
- try (OzoneClient rpcClient = createOzoneClient(omServiceID,
- ozoneConfiguration)) {
+ try (OzoneClient rpcClient = createOzoneClient(omServiceID, conf)) {
ensureVolumeAndBucketExist(rpcClient, volumeName, bucketName);
- ozoneManagerClient = createOmClient(ozoneConfiguration, omServiceID);
+ ozoneManagerClient = createOmClient(conf, omServiceID);
timer = getMetrics().timer("key-create");
@@ -107,8 +105,8 @@ public class OmKeyGenerator extends BaseFreonGenerator
OmKeyArgs keyArgs = new Builder()
.setBucketName(bucketName)
.setVolumeName(volumeName)
- .setReplicationConfig(RatisReplicationConfig.getInstance(factor))
.setKeyName(generateObjectName(counter))
+ .setReplicationConfig(replicationConfig)
.setLocationInfoList(new ArrayList<>())
.setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroupNames(),
ALL, ALL))
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
index be1a27258a..74cd0d0b37 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
@@ -21,20 +21,16 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Callable;
-import org.apache.hadoop.fs.ozone.OzoneClientUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import com.codahale.metrics.Timer;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
-import picocli.CommandLine.Spec;
/**
* Data generator tool test om performance.
@@ -48,8 +44,6 @@ import picocli.CommandLine.Spec;
public class OzoneClientKeyGenerator extends BaseFreonGenerator
implements Callable<Void> {
- @Spec private CommandSpec spec;
-
@Option(names = {"-v", "--volume"},
description = "Name of the bucket which contains the test data. Will be"
+ " created if missing.",
@@ -72,30 +66,13 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator
defaultValue = "4096")
private int bufferSize;
- @Option(names = { "-F", "--factor" },
- description = "[Deprecated] Replication factor (ONE, THREE)",
- defaultValue = "THREE"
- )
- private ReplicationFactor factor = ReplicationFactor.THREE;
-
@Option(names = "--om-service-id",
description = "OM Service ID"
)
private String omServiceID;
- @Option(names = {"--replication"},
- description = "Replication level of the new keys. Example: THREE"
- + " (for RATIS) or ONE (for STAND_ALONE). In case of EC, pass"
- + " CODEC-DATA-PARITY-CHUNKSIZE, eg rs-3-2-1024k, rs-6-3-1024k,"
- + " rs-10-4-1024k. Defaults to bucket or cluster config"
- )
- private String replication;
-
- @Option(names = {"--type"},
- description = "Replication type of the new key. Supported types are"
- + " RATIS, STANDALONE, EC. Defaults to bucket or cluster config"
- )
- private ReplicationType type;
+ @Mixin
+ private FreonReplicationOptions replication;
private Timer timer;
@@ -114,14 +91,7 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator
contentGenerator = new ContentGenerator(keySize, bufferSize);
metadata = new HashMap<>();
- if (spec.commandLine().getParseResult().hasMatchedOption("--factor")) {
- replicationConfig = ReplicationConfig
- .fromTypeAndFactor(ReplicationType.RATIS, factor);
- } else {
- replicationConfig = OzoneClientUtils
- .validateAndGetClientReplicationConfig(type, replication,
- ozoneConfiguration);
- }
+ replicationConfig = replication.fromParamsOrConfig(ozoneConfiguration);
try (OzoneClient rpcClient = createOzoneClient(omServiceID,
ozoneConfiguration)) {
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 07748f040c..75b17434e6 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -38,13 +38,10 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.LongSupplier;
-import org.apache.hadoop.fs.ozone.OzoneClientUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -74,6 +71,7 @@ import org.apache.commons.lang3.time.DurationFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
import picocli.CommandLine.ParentCommand;
@@ -179,26 +177,8 @@ public final class RandomKeyGenerator implements Callable<Void> {
)
private String jsonDir;
- @Option(
- names = {"--type", "--replication-type", "--replicationType"},
- description = "Replication type (STAND_ALONE, RATIS, EC). Full name " +
- "--replicationType will be removed in later versions."
- )
- private ReplicationType type;
-
- @Option(names = {"--replication"},
- description =
- "Replication configuration of the new key."
- + "(ONE, THREE) for RATIS or STAND_ALONE, "
- + "(rs-3-2-1024k, rs-6-3-1024k or rs-10-4-1024k) for EC."
- )
- private String replication;
-
- @Option(
- names = "--factor",
- description = "[Deprecated] Replication factor (ONE, THREE)"
- )
- private ReplicationFactor factor;
+ @Mixin
+ private FreonReplicationOptions replication;
@Option(
names = "--om-service-id",
@@ -258,6 +238,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ProgressBar progressbar;
RandomKeyGenerator() {
+ // for picocli
}
@VisibleForTesting
@@ -312,17 +293,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
init(ozoneConfiguration);
- if (factor != null) {
- // for backward compatibility
- if (type == null) {
- type = ReplicationType.STAND_ALONE;
- }
- replicationConfig = ReplicationConfig.fromTypeAndFactor(type, factor);
- } else {
- replicationConfig = OzoneClientUtils
- .validateAndGetClientReplicationConfig(type, replication,
- ozoneConfiguration);
- }
+ replicationConfig = replication.fromParamsOrConfig(ozoneConfiguration);
keyValueBuffer = StringUtils.string2Bytes(
RandomStringUtils.randomAscii(bufferSize));
@@ -957,7 +928,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
private int numOfThreads;
private String dataWritten;
private String execTime;
- private String replicationFactor;
+ private String replication;
private String replicationType;
private long keySize;
@@ -990,8 +961,8 @@ public final class RandomKeyGenerator implements Callable<Void> {
this.keySize = RandomKeyGenerator.this.keySize;
this.bufferSize = RandomKeyGenerator.this.bufferSize;
this.jobStartTime = Time.formatTime(RandomKeyGenerator.this.jobStartTime);
- this.replicationFactor = RandomKeyGenerator.this.factor.name();
- this.replicationType = RandomKeyGenerator.this.type.name();
+ replicationType = replicationConfig.getReplicationType().name();
+ replication = replicationConfig.getReplication();
long totalBytes =
(long) numOfVolumes * numOfBuckets * numOfKeys * keySize;
@@ -1128,8 +1099,8 @@ public final class RandomKeyGenerator implements Callable<Void> {
return execTime;
}
- public String getReplicationFactor() {
- return replicationFactor;
+ public String getReplication() {
+ return replication;
}
public String getReplicationType() {
@@ -1249,58 +1220,8 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
- @VisibleForTesting
- public void setNumOfVolumes(int numOfVolumes) {
- this.numOfVolumes = numOfVolumes;
- }
-
- @VisibleForTesting
- public void setNumOfBuckets(int numOfBuckets) {
- this.numOfBuckets = numOfBuckets;
- }
-
- @VisibleForTesting
- public void setNumOfKeys(int numOfKeys) {
- this.numOfKeys = numOfKeys;
- }
-
- @VisibleForTesting
- public void setNumOfThreads(int numOfThreads) {
- this.numOfThreads = numOfThreads;
- }
-
- @VisibleForTesting
- public void setKeySize(long keySize) {
- this.keySize = keySize;
- }
-
- @VisibleForTesting
- public void setType(ReplicationType type) {
- this.type = type;
- }
-
- @VisibleForTesting
- public void setReplication(String replication) {
- this.replication = replication;
- }
-
- @VisibleForTesting
- public void setFactor(ReplicationFactor factor) {
- this.factor = factor;
- }
-
- @VisibleForTesting
- public void setValidateWrites(boolean validateWrites) {
- this.validateWrites = validateWrites;
- }
-
@VisibleForTesting
public int getThreadPoolSize() {
return threadPoolSize;
}
-
- @VisibleForTesting
- public void setCleanObjects(boolean cleanObjects) {
- this.cleanObjects = cleanObjects;
- }
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
new file mode 100644
index 0000000000..bb4e0a7e2e
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.fs.ozone.OzoneClientUtils;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import picocli.CommandLine;
+
+import java.util.Optional;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
+
+/**
+ * Common options for specifying replication config: specialized for
+ * Ozone Shell and Freon commands.
+ */
+public abstract class ReplicationOptions {
+
+ private ReplicationType type;
+ private String replication;
+
+ public Optional<ReplicationConfig> fromParams(ConfigurationSource conf) {
+ if (replication == null && type == null) {
+ return Optional.empty();
+ }
+
+ // special case for backward compatibility: factor from config for RATIS
+ if (ReplicationType.RATIS == type && replication == null) {
+ replication = conf.get(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT);
+ }
+
+ return Optional.of(
+ ReplicationConfig.parseWithoutFallback(type, replication, conf));
+ }
+
+ public Optional<ReplicationConfig> fromConfig(ConfigurationSource conf) {
+ ReplicationType defaultType = Optional
+ .ofNullable(conf.get(OZONE_REPLICATION_TYPE))
+ .map(ReplicationType::valueOf)
+ .orElse(null);
+ String defaultReplication =
+ conf.get(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT);
+
+ return Optional.ofNullable(
+ OzoneClientUtils.validateAndGetClientReplicationConfig(
+ defaultType, defaultReplication, conf));
+ }
+
+ public ReplicationConfig fromParamsOrConfig(ConfigurationSource conf) {
+ return fromParams(conf)
+ .orElseGet(() -> fromConfig(conf)
+ .orElse(null));
+ }
+
+ @CommandLine.Option(names = {"--replication", "-r"},
+ description = "Replication definition. Valid values are replication"
+ + " type-specific. For RATIS: ONE or THREE."
+ + " In case of EC, pass CODEC-DATA-PARITY-CHUNKSIZE, "
+ + " e.g. rs-3-2-1024k, rs-6-3-1024k, rs-10-4-1024k")
+ public void setReplication(String replication) {
+ this.replication = replication;
+ }
+
+ // Option is defined in subclasses
+ public void setType(ReplicationType type) {
+ this.type = type;
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/SetReplicationConfigOptions.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/SetReplicationConfigOptions.java
deleted file mode 100644
index 354f629348..0000000000
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/SetReplicationConfigOptions.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.shell;
-
-import picocli.CommandLine;
-
-/**
- * Common options for set replication config.
- */
-public class SetReplicationConfigOptions {
- @CommandLine.Option(names = {"--replication",
- "-r"}, description = "Replication value. Example: 3 (for Ratis type)"
- + " or 1 ( for" + " standalone type). In the case of EC, pass"
- + " DATA-PARITY, eg 3-2," + " 6-3, 10-4")
- private String replication;
-
- @CommandLine.Option(names = {"--type",
- "-t"}, description = "Replication type. Supported types are RATIS,"
- + " STAND_ALONE, EC")
- private String replicationType;
-
- public String getReplication() {
- return this.replication;
- }
-
- public String getType() {
- return this.replicationType;
- }
-}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java
new file mode 100644
index 0000000000..3ee20c8b39
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.hadoop.hdds.client.ReplicationType;
+import picocli.CommandLine.Option;
+
+/**
+ * Options for specifying replication config in 'ozone shell' commands.
+ */
+public class ShellReplicationOptions extends ReplicationOptions {
+
+ @Option(names = {"-t", "--type", "--replication-type"},
+ description = "Replication type. Supported types are: RATIS, EC")
+ @Override
+ public void setType(ReplicationType type) {
+ super.setType(type);
+ }
+}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index b98676957e..ea66429fec 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -19,14 +19,8 @@ package org.apache.hadoop.ozone.shell.bucket;
import com.google.common.base.Strings;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
-import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -36,6 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.shell.OzoneAddress;
import org.apache.hadoop.ozone.shell.SetSpaceQuotaOptions;
+import org.apache.hadoop.ozone.shell.ShellReplicationOptions;
import org.apache.hadoop.security.UserGroupInformation;
import picocli.CommandLine;
import picocli.CommandLine.Command;
@@ -71,17 +66,8 @@ public class CreateBucketHandler extends BucketHandler {
defaultValue = "OBJECT_STORE")
private AllowedBucketLayouts allowedBucketLayout;
- @Option(names = {"--replication", "-r"},
- description = "Default replication level. Example: THREE"
- + " (for RATIS) or ONE (for STAND_ALONE). In case of EC, pass"
- + " CODEC-DATA-PARITY-CHUNKSIZE, eg rs-3-2-1024k, rs-6-3-1024k,"
- + " rs-10-4-1024k")
- private String replication;
-
- @Option(names = {"--type", "-t"},
- description = "Default replication type. Supported types are"
- + " RATIS, STANDALONE, EC")
- private String replicationType;
+ @CommandLine.Mixin
+ private ShellReplicationOptions replication;
@CommandLine.Mixin
private SetSpaceQuotaOptions quotaOptions;
@@ -122,25 +108,8 @@ public class CreateBucketHandler extends BucketHandler {
}
}
- if (replicationType != null) {
- if (replication != null) {
- ReplicationConfig replicationConfig = ReplicationConfig
- .parse(ReplicationType.valueOf(replicationType),
- replication, new OzoneConfiguration());
- boolean isEC = replicationConfig
- .getReplicationType() == HddsProtos.ReplicationType.EC;
- bb.setDefaultReplicationConfig(new DefaultReplicationConfig(
- ReplicationType.fromProto(replicationConfig.getReplicationType()),
- isEC ?
- null :
- ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()),
- isEC ? (ECReplicationConfig) replicationConfig : null));
- } else {
- throw new IOException(
- "Replication can't be null. Replication type passed was : "
- + replicationType);
- }
- }
+ replication.fromParams(getConf()).ifPresent(config ->
+ bb.setDefaultReplicationConfig(new DefaultReplicationConfig(config)));
if (!Strings.isNullOrEmpty(quotaOptions.getQuotaInBytes())) {
bb.setQuotaInBytes(OzoneQuota.parseSpaceQuota(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
index f5ad3a678a..45d66fd1c3 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
@@ -17,16 +17,13 @@
*/
package org.apache.hadoop.ozone.shell.bucket;
-import com.google.common.base.Strings;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneIllegalArgumentException;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.shell.OzoneAddress;
-import org.apache.hadoop.ozone.shell.SetReplicationConfigOptions;
+import org.apache.hadoop.ozone.shell.ShellReplicationOptions;
import picocli.CommandLine;
import java.io.IOException;
@@ -38,24 +35,20 @@ import java.io.IOException;
description = "Set replication config on bucket")
public class SetReplicationConfigHandler extends BucketHandler {
- @CommandLine.Mixin private SetReplicationConfigOptions
- replicationConfigOptions;
+ @CommandLine.Mixin
+ private ShellReplicationOptions replication;
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException, OzoneClientException {
- if (Strings.isNullOrEmpty(replicationConfigOptions.getType()) || Strings
- .isNullOrEmpty(replicationConfigOptions.getReplication())) {
- throw new OzoneIllegalArgumentException(
- "Replication type or replication factor cannot be null.");
- }
+ ReplicationConfig replicationConfig = replication.fromParams(getConf())
+ .orElseThrow(() -> new OzoneIllegalArgumentException(
+ "Replication type and config must be specified."));
+
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
OzoneBucket bucket =
client.getObjectStore().getVolume(volumeName).getBucket(bucketName);
- bucket.setReplicationConfig(ReplicationConfig
- .parse(ReplicationType.valueOf(replicationConfigOptions.getType()),
- replicationConfigOptions.getReplication(),
- new OzoneConfiguration()));
+ bucket.setReplicationConfig(replicationConfig);
}
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/CopyKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/CopyKeyHandler.java
index 5c0aaa3b90..81d2dbcae6 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/CopyKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/CopyKeyHandler.java
@@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -34,12 +33,14 @@ import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.apache.hadoop.ozone.shell.ShellReplicationOptions;
import org.apache.hadoop.ozone.shell.bucket.BucketHandler;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
+import picocli.CommandLine.Mixin;
import picocli.CommandLine.Parameters;
/**
@@ -58,17 +59,8 @@ public class CopyKeyHandler extends BucketHandler {
description = "The new desired name of the key")
private String toKey;
- @Option(names = {"-r", "--replication"},
- description =
- "Replication configuration of the new key. (this is replication "
- + "specific. for RATIS/STANDALONE you can use ONE or THREE) "
- + "Default is specified in the cluster-wide config.")
- private String replication;
-
- @Option(names = {"-t", "--type"},
- description = "Replication type of the new key. (use RATIS or " +
- "STAND_ALONE) Default is specified in the cluster-wide config.")
- private ReplicationType replicationType;
+ @Mixin
+ private ShellReplicationOptions replication;
@Override
protected void execute(OzoneClient client, OzoneAddress address)
@@ -81,7 +73,7 @@ public class CopyKeyHandler extends BucketHandler {
OzoneBucket bucket = vol.getBucket(bucketName);
ReplicationConfig replicationConfig =
- ReplicationConfig.parse(replicationType, replication, getConf());
+ replication.fromParamsOrConfig(getConf());
OzoneKeyDetails keyDetail = bucket.getKey(fromKey);
Map<String, String> keyMetadata = new HashMap<>(keyDetail.getMetadata());
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index f29227bdfb..7d7885d168 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -27,9 +27,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.fs.ozone.OzoneClientUtils;
import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -42,8 +40,9 @@ import org.apache.commons.codec.digest.DigestUtils;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+import org.apache.hadoop.ozone.shell.ShellReplicationOptions;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
+import picocli.CommandLine.Mixin;
import picocli.CommandLine.Parameters;
/**
@@ -56,17 +55,8 @@ public class PutKeyHandler extends KeyHandler {
@Parameters(index = "1", arity = "1..1", description = "File to upload")
private String fileName;
- @Option(names = {"-r", "--replication"},
- description = "Replication level of the new key. Example: THREE"
- + " (for RATIS) or ONE (for STAND_ALONE). In case of EC, pass"
- + " CODEC-DATA-PARITY-CHUNKSIZE, eg rs-3-2-1024k, rs-6-3-1024k,"
- + " rs-10-4-1024k. Defaults to bucket or cluster config")
- private String replication;
-
- @Option(names = {"-t", "--type"},
- description = "Replication type of the new key. Supported types are"
- + " RATIS, STANDALONE, EC. Defaults to bucket or cluster config")
- private ReplicationType replicationType;
+ @Mixin
+ private ShellReplicationOptions replication;
@Override
protected void execute(OzoneClient client, OzoneAddress address)
@@ -85,9 +75,8 @@ public class PutKeyHandler extends KeyHandler {
}
}
- ReplicationConfig replicationConfig = OzoneClientUtils
- .validateAndGetClientReplicationConfig(replicationType, replication,
- getConf());
+ ReplicationConfig replicationConfig =
+ replication.fromParamsOrConfig(getConf());
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
OzoneBucket bucket = vol.getBucket(bucketName);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org