You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/01/14 05:11:33 UTC
[iotdb] branch master updated: [IOTDB-2384]Enhanced data type check on creating schema template (#4800)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b981551 [IOTDB-2384]Enhanced data type check on creating schema template (#4800)
b981551 is described below
commit b981551b0c34aa2cc36b80cc54cf53faa30b7c31
Author: ZhaoXin <x_...@163.com>
AuthorDate: Fri Jan 14 13:10:16 2022 +0800
[IOTDB-2384]Enhanced data type check on creating schema template (#4800)
---
.../org/apache/iotdb/db/metadata/MManager.java | 28 ++++++--
.../org/apache/iotdb/db/metadata/mtree/MTree.java | 6 +-
.../iotdb/db/metadata/MManagerBasicTest.java | 75 ++++++++++++++++++++++
3 files changed, 102 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index bc8be27..f6eb834 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -1869,16 +1869,16 @@ public class MManager {
for (String measurementId : measurementList) {
PartialPath fullPath = devicePath.concatNode(measurementId);
int index = mtree.getMountedNodeIndexOnMeasurementPath(fullPath);
- if (index != fullPath.getNodeLength() - 1) {
+ if ((index != fullPath.getNodeLength() - 1) && !mountedNodeFound) {
// this measurement is in template, need to assure mounted node exists and set using
// template.
- if (!mountedNodeFound) {
- // Without allowing overlap of template and MTree, this block run only once
- String[] mountedPathNodes = Arrays.copyOfRange(fullPath.getNodes(), 0, index + 1);
- IMNode mountedNode = getDeviceNodeWithAutoCreate(new PartialPath(mountedPathNodes));
+ // Without allowing overlap of template and MTree, this block run only once
+ String[] mountedPathNodes = Arrays.copyOfRange(fullPath.getNodes(), 0, index + 1);
+ IMNode mountedNode = getDeviceNodeWithAutoCreate(new PartialPath(mountedPathNodes));
+ if (!mountedNode.isUseTemplate()) {
setUsingSchemaTemplate(mountedNode);
- mountedNodeFound = true;
}
+ mountedNodeFound = true;
}
}
// get logical device node, may be in template. will be multiple if overlap is allowed.
@@ -2079,6 +2079,15 @@ public class MManager {
// region Interfaces and Implementation for Template operations
public void createSchemaTemplate(CreateTemplatePlan plan) throws MetadataException {
try {
+
+ List<List<TSDataType>> dataTypes = plan.getDataTypes();
+ List<List<TSEncoding>> encodings = plan.getEncodings();
+ for (int i = 0; i < dataTypes.size(); i++) {
+ for (int j = 0; j < dataTypes.get(i).size(); j++) {
+ SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i).get(j), encodings.get(i).get(j));
+ }
+ }
+
templateManager.createSchemaTemplate(plan);
// write wal
if (!isRecovering) {
@@ -2091,6 +2100,13 @@ public class MManager {
public void appendSchemaTemplate(AppendTemplatePlan plan) throws MetadataException {
try {
+
+ List<TSDataType> dataTypes = plan.getDataTypes();
+ List<TSEncoding> encodings = plan.getEncodings();
+ for (int idx = 0; idx < dataTypes.size(); idx++) {
+ SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(idx), encodings.get(idx));
+ }
+
templateManager.appendSchemaTemplate(plan);
// write wal
if (!isRecovering) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
index 22ca3b2..e8eda3e 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
@@ -1585,6 +1585,9 @@ public class MTree implements Serializable {
upperTemplate = cur.getSchemaTemplate() != null ? cur.getSchemaTemplate() : upperTemplate;
if (!cur.hasChild(fullPathNodes[index])) {
if (upperTemplate != null) {
+ // for this fullPath, cur is the last node on MTree
+ // since upperTemplate exists, need to find the matched suffix path of fullPath and
+ // template
String suffixPath =
new PartialPath(Arrays.copyOfRange(fullPathNodes, index, fullPathNodes.length))
.toString();
@@ -1595,7 +1598,8 @@ public class MTree implements Serializable {
return index - 1;
}
- // overlap with template, cast exception for now
+ // if suffix doesn't match, but first node name matched, it's an overlap with template
+ // cast exception for now
if (upperTemplate.getDirectNode(fullPathNodes[index]) != null) {
throw new TemplateImcompatibeException(
measurementPath.getFullPath(), upperTemplate.getName(), fullPathNodes[index]);
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 0749aa6..25c2793 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.metadata.utils.MetaUtils;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
+import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
@@ -920,6 +921,80 @@ public class MManagerBasicTest {
}
@Test
+ public void testTemplateWithUnsupportedTypeEncoding() throws MetadataException {
+ CreateTemplatePlan plan;
+ List<List<String>> measurementList = new ArrayList<>();
+ measurementList.add(Collections.singletonList("d1.s1"));
+ measurementList.add(Collections.singletonList("s2"));
+ measurementList.add(Arrays.asList("GPS.x", "GPS.y"));
+
+ List<List<TSDataType>> dataTypeList = new ArrayList<>();
+ dataTypeList.add(Collections.singletonList(TSDataType.INT32));
+ dataTypeList.add(Collections.singletonList(TSDataType.INT32));
+ dataTypeList.add(Arrays.asList(TSDataType.TEXT, TSDataType.FLOAT));
+
+ List<List<TSEncoding>> encodingList = new ArrayList<>();
+ encodingList.add(Collections.singletonList(TSEncoding.GORILLA));
+ encodingList.add(Collections.singletonList(TSEncoding.GORILLA));
+ encodingList.add(Arrays.asList(TSEncoding.RLE, TSEncoding.RLE));
+
+ List<List<CompressionType>> compressionTypes = new ArrayList<>();
+ compressionTypes.add(Collections.singletonList(CompressionType.SDT));
+ compressionTypes.add(Collections.singletonList(CompressionType.SNAPPY));
+ compressionTypes.add(Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY));
+
+ try {
+ plan =
+ new CreateTemplatePlan(
+ "treeTemplate", measurementList, dataTypeList, encodingList, compressionTypes);
+ IoTDB.metaManager.createSchemaTemplate(plan);
+ } catch (MetadataException e) {
+ assertEquals("encoding RLE does not support TEXT", e.getMessage());
+ }
+
+ dataTypeList.get(2).set(0, TSDataType.FLOAT);
+ CreateTemplatePlan planb =
+ new CreateTemplatePlan(
+ "treeTemplate", measurementList, dataTypeList, encodingList, compressionTypes);
+
+ IoTDB.metaManager.createSchemaTemplate(planb);
+ Template template = IoTDB.metaManager.getTemplate("treeTemplate");
+ assertEquals("[d1.s1, GPS.x, GPS.y, s2]", template.getAllMeasurementsPaths().toString());
+
+ List<String> appendMeasurements = Arrays.asList("a1", "a2");
+ List<TSDataType> appendDataTypes = Arrays.asList(TSDataType.TEXT, TSDataType.FLOAT);
+ List<TSEncoding> appendEncodings = Arrays.asList(TSEncoding.RLE, TSEncoding.RLE);
+ List<CompressionType> appendCompressor =
+ Arrays.asList(CompressionType.SNAPPY, CompressionType.LZ4);
+ AppendTemplatePlan plana =
+ new AppendTemplatePlan(
+ "treeTemplate",
+ false,
+ appendMeasurements,
+ appendDataTypes,
+ appendEncodings,
+ appendCompressor);
+ try {
+ IoTDB.metaManager.appendSchemaTemplate(plana);
+ } catch (MetadataException e) {
+ assertEquals("encoding RLE does not support TEXT", e.getMessage());
+ }
+
+ appendDataTypes.set(0, TSDataType.FLOAT);
+ AppendTemplatePlan planab =
+ new AppendTemplatePlan(
+ "treeTemplate",
+ false,
+ appendMeasurements,
+ appendDataTypes,
+ appendEncodings,
+ appendCompressor);
+ IoTDB.metaManager.appendSchemaTemplate(planab);
+ assertEquals(
+ "[a1, a2, d1.s1, GPS.x, GPS.y, s2]", template.getAllMeasurementsPaths().toString());
+ }
+
+ @Test
public void testTemplateInnerTree() {
CreateTemplatePlan plan = getTreeTemplatePlan();
Template template;