You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2022/05/23 03:23:10 UTC
[ozone] branch HDDS-3630 updated: HDDS-6747. Support configurable separator between container ID and container meta data key (#3420)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3630 by this push:
new 9b3de4101b HDDS-6747. Support configurable separator between container ID and container meta data key (#3420)
9b3de4101b is described below
commit 9b3de4101b5768b300f9c65b5c8a3f953ee23442
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Mon May 23 11:23:05 2022 +0800
HDDS-6747. Support configurable separator between container ID and container meta data key (#3420)
---
.../common/statemachine/DatanodeConfiguration.java | 17 ++++++++++++++
.../container/keyvalue/KeyValueContainerData.java | 3 +--
.../metadata/DatanodeSchemaThreeDBDefinition.java | 13 ++++++++++-
.../keyvalue/ContainerTestVersionInfo.java | 3 +++
.../keyvalue/TestKeyValueBlockIterator.java | 27 +++++++++++++++++-----
5 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index d6f59d3f7d..3a11c53289 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -271,6 +271,15 @@ public class DatanodeConfiguration {
private boolean containerSchemaV3Enabled =
CONTAINER_SCHEMA_V3_ENABLED_DEFAULT;
+ @Config(key = "container.schema.v3.key.separator",
+ defaultValue = "|",
+ type = ConfigType.STRING,
+ tags = { DATANODE },
+ description = "The default separator between Container ID and container" +
+ " meta key name."
+ )
+ private String containerSchemaV3KeySeparator = "|";
+
@PostConstruct
public void validate() {
if (containerDeleteThreads < 1) {
@@ -421,4 +430,12 @@ public class DatanodeConfiguration {
public void setContainerSchemaV3Enabled(boolean containerSchemaV3Enabled) {
this.containerSchemaV3Enabled = containerSchemaV3Enabled;
}
+
+ public String getContainerSchemaV3KeySeparator() {
+ return this.containerSchemaV3KeySeparator;
+ }
+
+ public void setContainerSchemaV3KeySeparator(String separator) {
+ this.containerSchemaV3KeySeparator = separator;
+ }
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index c8e4e60573..440161541d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -364,8 +364,7 @@ public class KeyValueContainerData extends ContainerData {
public KeyPrefixFilter getUnprefixedKeyFilter() {
String schemaPrefix = containerPrefix();
- return new KeyPrefixFilter().addFilter(
- schemaPrefix == null ? "#" : schemaPrefix + "#", true);
+ return new KeyPrefixFilter().addFilter(schemaPrefix + "#", true);
}
public KeyPrefixFilter getDeletingBlockKeyFilter() {
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
index 5286056119..ae4de8c13b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.utils.db.LongCodec;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.utils.db.DatanodeDBProfile;
import org.rocksdb.ColumnFamilyOptions;
@@ -87,10 +88,15 @@ public class DatanodeSchemaThreeDBDefinition
DeletedBlocksTransaction.class,
new DeletedBlocksTransactionCodec());
+ private static String separator = "";
+
public DatanodeSchemaThreeDBDefinition(String dbPath,
ConfigurationSource config) {
super(dbPath, config);
+ DatanodeConfiguration dc = config.getObject(DatanodeConfiguration.class);
+ setSeparator(dc.getContainerSchemaV3KeySeparator());
+
// Get global ColumnFamilyOptions first.
DatanodeDBProfile dbProfile = DatanodeDBProfile
.getProfile(config.getEnum(HDDS_DB_PROFILE, HDDS_DEFAULT_DB_PROFILE));
@@ -137,11 +143,16 @@ public class DatanodeSchemaThreeDBDefinition
public static String getContainerKeyPrefix(long containerID) {
// NOTE: Rocksdb normally needs a fixed length prefix.
- return FixedLengthStringUtils.bytes2String(Longs.toByteArray(containerID));
+ return FixedLengthStringUtils.bytes2String(Longs.toByteArray(containerID))
+ + separator;
}
private static int getContainerKeyPrefixLength() {
return FixedLengthStringUtils.string2Bytes(
getContainerKeyPrefix(0L)).length;
}
+
+ private void setSeparator(String keySeparator) {
+ separator = keySeparator;
+ }
}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
index 7111374b2b..c1292ea46c 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerTestVersionInfo.java
@@ -65,6 +65,9 @@ public class ContainerTestVersionInfo {
.collect(toList());
}
+ public static List<ContainerTestVersionInfo> getLayoutList() {
+ return layoutList;
+ }
public static void setTestSchemaVersion(String schemaVersion,
OzoneConfiguration conf) {
if (schemaVersion.equals(OzoneConsts.SCHEMA_V3)) {
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index a68349c0aa..3734c5b1f5 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -41,12 +41,14 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.interfaces.BlockIterator;
import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.ozone.test.GenericTestUtils;
+import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import org.junit.After;
@@ -79,16 +81,31 @@ public class TestKeyValueBlockIterator {
private String datanodeID = UUID.randomUUID().toString();
private String clusterID = UUID.randomUUID().toString();
- public TestKeyValueBlockIterator(ContainerTestVersionInfo versionInfo) {
+ public TestKeyValueBlockIterator(ContainerTestVersionInfo versionInfo,
+ String keySeparator) {
this.layout = versionInfo.getLayout();
this.schemaVersion = versionInfo.getSchemaVersion();
this.conf = new OzoneConfiguration();
ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
+ DatanodeConfiguration dc = conf.getObject(DatanodeConfiguration.class);
+ dc.setContainerSchemaV3KeySeparator(keySeparator);
+ conf.setFromObject(dc);
}
@Parameterized.Parameters
public static Iterable<Object[]> data() {
- return ContainerTestVersionInfo.versionParameters();
+ List listA =
+ ContainerTestVersionInfo.getLayoutList().stream().map(
+ each -> new Object[] {each, ""})
+ .collect(toList());
+ List listB =
+ ContainerTestVersionInfo.getLayoutList().stream().map(
+ each -> new Object[] {each,
+ new DatanodeConfiguration().getContainerSchemaV3KeySeparator()})
+ .collect(toList());
+
+ listB.addAll(listA);
+ return listB;
}
@Before
@@ -290,8 +307,7 @@ public class TestKeyValueBlockIterator {
String schemaPrefix = containerData.containerPrefix();
MetadataKeyFilters.KeyPrefixFilter secondFilter =
new MetadataKeyFilters.KeyPrefixFilter()
- .addFilter(schemaPrefix == null ?
- secondPrefix : schemaPrefix + secondPrefix);
+ .addFilter(schemaPrefix + secondPrefix);
testWithFilter(secondFilter, blockIDs.get(secondPrefix));
}
@@ -399,8 +415,7 @@ public class TestKeyValueBlockIterator {
blockIndex++;
BlockData blockData = new BlockData(blockID);
blockData.setChunks(chunkList);
- String blockKey = (schemaPrefix == null ? "" : schemaPrefix) +
- prefix + blockID.getLocalID();
+ String blockKey = schemaPrefix + prefix + blockID.getLocalID();
blockDataTable.put(blockKey, blockData);
blockIDs.get(prefix).add(blockID.getLocalID());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org