You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/05/04 08:16:10 UTC
[iotdb] branch master updated: [IOTDB-3088]Implement MTreeAboveSG serialize and deserialize (#5781)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b090698af2 [IOTDB-3088]Implement MTreeAboveSG serialize and deserialize (#5781)
b090698af2 is described below
commit b090698af2eb0bbec3082c0ece78a3de39891ba3
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Wed May 4 16:16:04 2022 +0800
[IOTDB-3088]Implement MTreeAboveSG serialize and deserialize (#5781)
---
.../iotdb/db/metadata/mtree/MTreeAboveSG.java | 86 +++++++++++++++++++++-
.../iotdb/db/metadata/mtree/MTreeAboveSGTest.java | 47 ++++++++++++
2 files changed, 131 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSG.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSG.java
index f5da8fed97..c40ecdf047 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSG.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSG.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
@@ -43,6 +44,10 @@ import org.apache.iotdb.db.metadata.mtree.traverser.counter.StorageGroupCounter;
import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.metadata.utils.MetaFormatUtils;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.ArrayDeque;
@@ -53,15 +58,20 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Stack;
import java.util.TreeSet;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT;
+import static org.apache.iotdb.db.metadata.MetadataConstant.INTERNAL_MNODE_TYPE;
+import static org.apache.iotdb.db.metadata.MetadataConstant.STORAGE_GROUP_MNODE_TYPE;
// Since the MTreeAboveSG is all stored in memory, thus it is not restricted to manage MNode through
// MTreeStore.
public class MTreeAboveSG {
+ private final Logger logger = LoggerFactory.getLogger(MTreeAboveSG.class);
+
private IMNode root;
private IMTreeStore store;
@@ -506,10 +516,82 @@ public class MTreeAboveSG {
}
public void serialize(ByteBuffer buffer) {
- // TODO: @yukun serialize MTreeAboveSG to ByteBuffer
+ serializeInternalNode((InternalMNode) this.root, buffer);
+ }
+
+ private void serializeInternalNode(InternalMNode node, ByteBuffer buffer) {
+ for (IMNode child : node.getChildren().values()) {
+ if (child.isStorageGroup()) {
+ serializeStorageGroupNode((StorageGroupMNode) child, buffer);
+ } else {
+ serializeInternalNode((InternalMNode) child, buffer);
+ }
+ }
+
+ buffer.put(INTERNAL_MNODE_TYPE);
+ ReadWriteIOUtils.write(node.getName(), buffer);
+ ReadWriteIOUtils.write(node.getChildren().size(), buffer);
+ }
+
+ private void serializeStorageGroupNode(StorageGroupMNode storageGroupNode, ByteBuffer buffer) {
+ buffer.put(STORAGE_GROUP_MNODE_TYPE);
+ ReadWriteIOUtils.write(storageGroupNode.getName(), buffer);
+ ThriftConfigNodeSerDeUtils.writeTStorageGroupSchema(
+ storageGroupNode.getStorageGroupSchema(), buffer);
}
public void deserialize(ByteBuffer buffer) {
- // TODO: @yukun deserialize MTreeAboveSG from ByteBuffer
+ byte type = buffer.get();
+ if (type != STORAGE_GROUP_MNODE_TYPE) {
+ logger.error("Wrong node type. Cannot deserialize MTreeAboveSG from given buffer");
+ return;
+ }
+
+ StorageGroupMNode storageGroupMNode = deserializeStorageGroupMNode(buffer);
+ InternalMNode internalMNode;
+
+ Stack<InternalMNode> stack = new Stack<>();
+ stack.push(storageGroupMNode);
+
+ String name = storageGroupMNode.getName();
+ int childNum = 0;
+
+ while (!PATH_ROOT.equals(name)) {
+ type = buffer.get();
+ switch (type) {
+ case INTERNAL_MNODE_TYPE:
+ internalMNode = deserializeInternalMNode(buffer);
+ childNum = ReadWriteIOUtils.readInt(buffer);
+ while (childNum > 0) {
+ internalMNode.addChild(stack.pop());
+ childNum--;
+ }
+ stack.push(internalMNode);
+ name = internalMNode.getName();
+ break;
+ case STORAGE_GROUP_MNODE_TYPE:
+ storageGroupMNode = deserializeStorageGroupMNode(buffer);
+ childNum = 0;
+ stack.push(storageGroupMNode);
+ name = storageGroupMNode.getName();
+ break;
+ default:
+ logger.error("Unrecognized node type. Cannot deserialize MTreeAboveSG from given buffer");
+ return;
+ }
+ }
+ this.root = stack.peek();
+ }
+
+ private InternalMNode deserializeInternalMNode(ByteBuffer buffer) {
+ return new InternalMNode(null, ReadWriteIOUtils.readString(buffer));
+ }
+
+ private StorageGroupMNode deserializeStorageGroupMNode(ByteBuffer buffer) {
+ StorageGroupMNode storageGroupMNode =
+ new StorageGroupMNode(null, ReadWriteIOUtils.readString(buffer));
+ storageGroupMNode.setStorageGroupSchema(
+ ThriftConfigNodeSerDeUtils.readTStorageGroupSchema(buffer));
+ return storageGroupMNode;
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSGTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSGTest.java
index 543f86b26c..4b8140a3f8 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSGTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeAboveSGTest.java
@@ -21,7 +21,9 @@ package org.apache.iotdb.db.metadata.mtree;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.confignode.rpc.thrift.TStorageGroupSchema;
import org.apache.iotdb.db.metadata.LocalSchemaProcessor;
+import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -30,6 +32,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -289,4 +292,48 @@ public class MTreeAboveSGTest {
Assert.assertEquals(0, result.left.size());
Assert.assertEquals(1, result.right.size());
}
+
+ @Test
+ public void testSerialization() throws Exception {
+ PartialPath[] pathList =
+ new PartialPath[] {
+ new PartialPath("root.sg"),
+ new PartialPath("root.a.sg"),
+ new PartialPath("root.a.b.sg"),
+ new PartialPath("root.a.a.b.sg")
+ };
+ for (int i = 0; i < pathList.length; i++) {
+ root.setStorageGroup(pathList[i]);
+ IStorageGroupMNode storageGroupMNode =
+ root.getStorageGroupNodeByStorageGroupPath(pathList[i]);
+ storageGroupMNode.setDataTTL(i);
+ storageGroupMNode.setDataReplicationFactor(i);
+ storageGroupMNode.setSchemaReplicationFactor(i);
+ storageGroupMNode.setTimePartitionInterval(i);
+ }
+
+ ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024);
+ root.serialize(byteBuffer);
+ byteBuffer.flip();
+
+ MTreeAboveSG newTree = new MTreeAboveSG();
+ newTree.deserialize(byteBuffer);
+
+ for (int i = 0; i < pathList.length; i++) {
+ newTree.isStorageGroup(pathList[i]);
+ TStorageGroupSchema storageGroupSchema =
+ newTree.getStorageGroupNodeByStorageGroupPath(pathList[i]).getStorageGroupSchema();
+ Assert.assertEquals(i, storageGroupSchema.getTTL());
+ Assert.assertEquals(i, storageGroupSchema.getSchemaReplicationFactor());
+ Assert.assertEquals(i, storageGroupSchema.getDataReplicationFactor());
+ Assert.assertEquals(i, storageGroupSchema.getTimePartitionInterval());
+ }
+
+ Assert.assertEquals(
+ 3, newTree.getMatchedStorageGroups(new PartialPath("root.**.sg"), false).size());
+ Assert.assertEquals(
+ 2, newTree.getMatchedStorageGroups(new PartialPath("root.**.b.sg"), false).size());
+ Assert.assertEquals(
+ 1, newTree.getMatchedStorageGroups(new PartialPath("root.*.*.sg"), false).size());
+ }
}