You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2023/05/16 12:33:25 UTC
[iotdb] branch master updated: [IOTDB-5884] Throw Exception when alter template with duplicate measurement (#9864)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e2aa814482 [IOTDB-5884] Throw Exception when alter template with duplicate measurement (#9864)
e2aa814482 is described below
commit e2aa814482b10948e5c95ffcb20ec1d33b69c055
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Tue May 16 20:33:17 2023 +0800
[IOTDB-5884] Throw Exception when alter template with duplicate measurement (#9864)
---
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 9 +++
.../session/it/IoTDBSessionSchemaTemplateIT.java | 92 ++++++++++++++++++++++
.../template/alter/TemplateExtendInfo.java | 35 ++++++++
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
.../config/executor/ClusterConfigTaskExecutor.java | 21 +++++
5 files changed, 162 insertions(+), 1 deletion(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
index ebf38d5543..ec0823e85b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBExtendTemplateIT.java
@@ -84,6 +84,15 @@ public class IoTDBExtendTemplateIT extends AbstractSchemaIT {
statement.execute(
"ALTER SCHEMA TEMPLATE t1 ADD(s3 INT64 ENCODING=RLE, s4 DOUBLE ENCODING=GORILLA)");
+ try {
+ statement.execute(
+ "ALTER SCHEMA TEMPLATE t1 ADD(s5 INT64 ENCODING=RLE, s5 DOUBLE ENCODING=GORILLA)");
+ } catch (SQLException e) {
+ Assert.assertTrue(
+ e.getMessage()
+ .contains("Duplicated measurement [s5] in schema template alter request"));
+ }
+
String[] sqls =
new String[] {
"show timeseries",
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSchemaTemplateIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSchemaTemplateIT.java
index d80a5fba0a..b71005bc0d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSchemaTemplateIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSchemaTemplateIT.java
@@ -406,4 +406,96 @@ public class IoTDBSessionSchemaTemplateIT extends AbstractSchemaIT {
Assert.assertTrue(expectedSeries.isEmpty());
}
+
+ @Test
+ public void testHybridAutoExtendSchemaTemplate()
+ throws StatementExecutionException, IoTDBConnectionException, IOException {
+ session.createDatabase("root.db");
+
+ Template temp1 = getTemplate("template1");
+ Template temp2 = new Template("template2", false);
+
+ assertEquals("[]", session.showAllTemplates().toString());
+
+ session.createSchemaTemplate(temp1);
+ session.createSchemaTemplate(temp2);
+
+ session.setSchemaTemplate("template1", "root.db.v1");
+
+ session.createTimeseriesUsingSchemaTemplate(Collections.singletonList("root.db.v1.d1"));
+
+ session.setSchemaTemplate("template2", "root.db.v4");
+
+ List<String> deviceIds =
+ Arrays.asList(
+ "root.db.v1.d1", "root.db.v1.d2", "root.db.v2.d1", "root.db.v4.d1", "root.db.v4.d2");
+ List<Long> timestamps = Arrays.asList(1L, 1L, 1L, 1L, 1L);
+ List<String> measurements = Arrays.asList("x", "y", "z");
+ List<List<String>> allMeasurements =
+ Arrays.asList(measurements, measurements, measurements, measurements, measurements);
+ List<TSDataType> tsDataTypes =
+ Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.TEXT);
+ List<List<TSDataType>> allTsDataTypes =
+ Arrays.asList(tsDataTypes, tsDataTypes, tsDataTypes, tsDataTypes, tsDataTypes);
+ List<Object> values = Arrays.asList(1f, 2f, "3");
+ List<List<Object>> allValues = Arrays.asList(values, values, values, values, values);
+
+ session.insertRecords(deviceIds, timestamps, allMeasurements, allTsDataTypes, allValues);
+
+ Set<String> expectedSeries =
+ new HashSet<>(
+ Arrays.asList(
+ "root.db.v1.d1.x",
+ "root.db.v1.d1.y",
+ "root.db.v1.d1.z",
+ "root.db.v1.d2.x",
+ "root.db.v1.d2.y",
+ "root.db.v1.d2.z",
+ "root.db.v2.d1.x",
+ "root.db.v2.d1.y",
+ "root.db.v2.d1.z",
+ "root.db.v4.d1.x",
+ "root.db.v4.d1.y",
+ "root.db.v4.d1.z",
+ "root.db.v4.d2.x",
+ "root.db.v4.d2.y",
+ "root.db.v4.d2.z"));
+
+ try (SessionDataSet dataSet = session.executeQueryStatement("show timeseries")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Assert.assertTrue(expectedSeries.contains(iterator.getString(1)));
+ expectedSeries.remove(iterator.getString(1));
+ }
+ }
+
+ Assert.assertTrue(expectedSeries.isEmpty());
+
+ deviceIds = Arrays.asList("root.db.v4.d1", "root.db.v4.d2");
+ timestamps = Arrays.asList(1L, 1L);
+ measurements = Arrays.asList("a", "b", "c");
+ allMeasurements = Arrays.asList(measurements, measurements);
+ allTsDataTypes =
+ Arrays.asList(
+ Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.TEXT),
+ Arrays.asList(TSDataType.DOUBLE, TSDataType.DOUBLE, TSDataType.INT32));
+ allValues = Arrays.asList(Arrays.asList(1f, 2f, "3"), Arrays.asList(1d, 2d, 3));
+
+ try {
+ session.insertRecords(deviceIds, timestamps, allMeasurements, allTsDataTypes, allValues);
+ } catch (StatementExecutionException e) {
+ Assert.assertTrue(
+ e.getMessage()
+ .contains(
+ "data type of root.db.v4.d2.a is not consistent, registered type FLOAT, inserting type DOUBLE"));
+ Assert.assertTrue(
+ e.getMessage()
+ .contains(
+ "data type of root.db.v4.d2.b is not consistent, registered type FLOAT, inserting type DOUBLE"));
+ Assert.assertTrue(
+ e.getMessage()
+ .contains(
+ "data type of root.db.v4.d2.c is not consistent, registered type TEXT, inserting type INT32"));
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/alter/TemplateExtendInfo.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/alter/TemplateExtendInfo.java
index 1ecdf0c878..71b28aa14e 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/template/alter/TemplateExtendInfo.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/alter/TemplateExtendInfo.java
@@ -28,7 +28,9 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
public class TemplateExtendInfo extends TemplateAlterInfo {
@@ -102,6 +104,39 @@ public class TemplateExtendInfo extends TemplateAlterInfo {
compressors.add(compressionType);
}
+ // if there's duplicate measurements, return the first one, otherwise return null
+ public String getFirstDuplicateMeasurement() {
+ if (measurements != null) {
+ Set<String> set = new HashSet<>();
+ for (String measurement : measurements) {
+ if (set.contains(measurement)) {
+ return measurement;
+ } else {
+ set.add(measurement);
+ }
+ }
+ }
+ return null;
+ }
+
+ // deduplicate the measurements with same name, keep the first one
+ public TemplateExtendInfo deduplicate() {
+ if (measurements == null || measurements.isEmpty()) {
+ return new TemplateExtendInfo();
+ }
+ Set<String> set = new HashSet<>();
+ TemplateExtendInfo result = new TemplateExtendInfo();
+ for (int i = 0; i < measurements.size(); i++) {
+ if (set.contains(measurements.get(i))) {
+ continue;
+ }
+ set.add(measurements.get(i));
+ result.addMeasurement(
+ measurements.get(i), dataTypes.get(i), encodings.get(i), compressors.get(i));
+ }
+ return result;
+ }
+
public void serialize(OutputStream outputStream) throws IOException {
super.serialize(outputStream);
ReadWriteIOUtils.write(measurements.size(), outputStream);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/AutoCreateSchemaExecutor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/AutoCreateSchemaExecutor.java
index 68f4b4782a..f45a068194 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/AutoCreateSchemaExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/AutoCreateSchemaExecutor.java
@@ -162,15 +162,17 @@ class AutoCreateSchemaExecutor {
}
}
+ // used for insert record or tablet
void autoExtendTemplate(
String templateName, List<String> measurementList, List<TSDataType> dataTypeList) {
internalExtendTemplate(templateName, measurementList, dataTypeList, null, null);
}
+ // used for insert records or tablets
void autoExtendTemplate(Map<String, TemplateExtendInfo> templateExtendInfoMap) {
TemplateExtendInfo templateExtendInfo;
for (Map.Entry<String, TemplateExtendInfo> entry : templateExtendInfoMap.entrySet()) {
- templateExtendInfo = entry.getValue();
+ templateExtendInfo = entry.getValue().deduplicate();
internalExtendTemplate(
entry.getKey(),
templateExtendInfo.getMeasurements(),
@@ -225,6 +227,7 @@ class AutoCreateSchemaExecutor {
}
}
+ // used for load TsFile
void autoCreateMissingMeasurements(
ClusterSchemaTree schemaTree,
List<PartialPath> devicePathList,
@@ -346,6 +349,7 @@ class AutoCreateSchemaExecutor {
if (!templateExtendInfoMap.isEmpty()) {
for (TemplateExtendInfo templateExtendInfo : templateExtendInfoMap.values()) {
+ templateExtendInfo = templateExtendInfo.deduplicate();
internalExtendTemplate(
templateExtendInfo.getTemplateName(),
templateExtendInfo.getMeasurements(),
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 5fb4f5b7b5..e1881628bb 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.ConfigRegionId;
import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.executable.ExecutableManager;
import org.apache.iotdb.commons.executable.ExecutableResource;
import org.apache.iotdb.commons.path.PartialPath;
@@ -104,7 +105,9 @@ import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.metadata.template.ClusterTemplateManager;
import org.apache.iotdb.db.metadata.template.Template;
+import org.apache.iotdb.db.metadata.template.TemplateAlterOperationType;
import org.apache.iotdb.db.metadata.template.alter.TemplateAlterOperationUtil;
+import org.apache.iotdb.db.metadata.template.alter.TemplateExtendInfo;
import org.apache.iotdb.db.mpp.plan.analyze.Analysis;
import org.apache.iotdb.db.mpp.plan.analyze.Analyzer;
import org.apache.iotdb.db.mpp.plan.execution.config.ConfigTaskResult;
@@ -1382,6 +1385,24 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
public SettableFuture<ConfigTaskResult> alterSchemaTemplate(
String queryId, AlterSchemaTemplateStatement alterSchemaTemplateStatement) {
SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+
+ if (alterSchemaTemplateStatement
+ .getOperationType()
+ .equals(TemplateAlterOperationType.EXTEND_TEMPLATE)) {
+ // check duplicate measurement
+ TemplateExtendInfo templateExtendInfo =
+ (TemplateExtendInfo) alterSchemaTemplateStatement.getTemplateAlterInfo();
+ String duplicateMeasurement = templateExtendInfo.getFirstDuplicateMeasurement();
+ if (duplicateMeasurement != null) {
+ future.setException(
+ new MetadataException(
+ String.format(
+ "Duplicated measurement [%s] in schema template alter request",
+ duplicateMeasurement)));
+ return future;
+ }
+ }
+
TAlterSchemaTemplateReq req = new TAlterSchemaTemplateReq();
req.setQueryId(queryId);
req.setTemplateAlterInfo(