You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/04/01 08:07:24 UTC
(iotdb) branch master updated: Fix Load TsFile with extend template & add template check for internalCreateTimeSeriesStatement (#12266)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2990412b8bb Fix Load TsFile with extend template & add template check for internalCreateTimeSeriesStatement (#12266)
2990412b8bb is described below
commit 2990412b8bb3e90c5d5c2f79cf52ce8540223ddd
Author: Chen YZ <43...@users.noreply.github.com>
AuthorDate: Mon Apr 1 16:07:18 2024 +0800
Fix Load TsFile with extend template & add template check for internalCreateTimeSeriesStatement (#12266)
---
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 67 ++++++++++++++++++++++
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 11 +++-
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
4 files changed, 81 insertions(+), 7 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
index b1eb54950d3..774c82e0ca2 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
@@ -50,8 +50,10 @@ import java.sql.Statement;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail;
@@ -231,6 +233,71 @@ public class IOTDBLoadTsFileIT {
}
}
+ @Test
+ public void testLoadWithExtendTemplate() throws Exception {
+ long writtenPoint1 = 0;
+ // device 0, device 1, sg 0
+ try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_0,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_00,
+ SchemaConfig.MEASUREMENT_01,
+ SchemaConfig.MEASUREMENT_02,
+ SchemaConfig.MEASUREMENT_03));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13));
+ generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false);
+ generator.generateData(SchemaConfig.DEVICE_1, 10000, PARTITION_INTERVAL / 10_000, true);
+ writtenPoint1 = generator.getTotalNumber();
+ }
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("create database root.sg.test_0");
+ statement.execute(
+ "create device template t1 (lat FLOAT encoding=Gorilla, lon FLOAT encoding=Gorilla)");
+ statement.execute(" set device template t1 to root.sg.test_0.d_0");
+
+ statement.execute(String.format("load \"%s\" sglevel=2", tmpDir.getAbsolutePath()));
+
+ try (ResultSet resultSet =
+ statement.executeQuery("select count(*) from root.** group by level=1,2")) {
+ if (resultSet.next()) {
+ long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ Assert.assertEquals(writtenPoint1, sg1Count);
+ } else {
+ Assert.fail("This ResultSet is empty.");
+ }
+ }
+
+ Set<String> nodes =
+ new HashSet<>(
+ Arrays.asList(
+ "lat",
+ "lon",
+ SchemaConfig.MEASUREMENT_00.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_01.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_02.getMeasurementId(),
+ SchemaConfig.MEASUREMENT_03.getMeasurementId()));
+ try (ResultSet resultSet = statement.executeQuery("show nodes in schema template t1")) {
+ while (resultSet.next()) {
+ String device = resultSet.getString(ColumnHeaderConstant.CHILD_NODES);
+ Assert.assertTrue(nodes.remove(device));
+ }
+ Assert.assertTrue(nodes.isEmpty());
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail("Parse result set error.");
+ }
+ }
+ }
+
@Test
public void testLoadWithAutoRegister() throws Exception {
long writtenPoint1 = 0;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 17b969d928c..3c7ae9e7a7c 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.path.PathPatternUtil;
import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest;
import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.service.metric.MetricService;
@@ -217,7 +218,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
-import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
/** Entry of all management, AssignPartitionManager, AssignRegionManager. */
@@ -1788,7 +1788,7 @@ public class ConfigManager implements IManager {
HashSet<TDatabaseSchema> deleteDatabaseSchemas = new HashSet<>();
List<PartialPath> deleteTimeSeriesPatternPaths = new ArrayList<>();
for (PartialPath path : rawPatternTree.getAllPathPatterns()) {
- if (path.getFullPath().endsWith(MULTI_LEVEL_PATH_WILDCARD)
+ if (PathPatternUtil.isMultiLevelMatchWildcard(path.getMeasurement())
&& !path.getDevicePath().hasWildcard()) {
Map<String, TDatabaseSchema> databaseSchemaMap =
getClusterSchemaManager().getMatchedDatabaseSchemasByPrefix(path.getDevicePath());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 9b97f03a090..f8f7c7ee12d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -65,6 +65,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.binary.CompareBinaryExpre
import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.MeasurementGroup;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.FillDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByConditionParameter;
@@ -2410,6 +2411,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Analysis analysis = new Analysis();
analysis.setStatement(internalCreateTimeSeriesStatement);
+ checkIsTemplateCompatible(
+ internalCreateTimeSeriesStatement.getDevicePath(),
+ internalCreateTimeSeriesStatement.getMeasurements(),
+ null);
PathPatternTree pathPatternTree = new PathPatternTree();
for (String measurement : internalCreateTimeSeriesStatement.getMeasurements()) {
@@ -2435,8 +2440,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analysis.setStatement(internalCreateMultiTimeSeriesStatement);
PathPatternTree pathPatternTree = new PathPatternTree();
- for (PartialPath devicePath : internalCreateMultiTimeSeriesStatement.getDeviceMap().keySet()) {
- pathPatternTree.appendFullPath(devicePath.concatNode(ONE_LEVEL_PATH_WILDCARD));
+ for (Map.Entry<PartialPath, Pair<Boolean, MeasurementGroup>> entry :
+ internalCreateMultiTimeSeriesStatement.getDeviceMap().entrySet()) {
+ checkIsTemplateCompatible(entry.getKey(), entry.getValue().right.getMeasurements(), null);
+ pathPatternTree.appendFullPath(entry.getKey().concatNode(ONE_LEVEL_PATH_WILDCARD));
}
SchemaPartition schemaPartitionInfo;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java
index cbf85d67a82..389e10638aa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java
@@ -402,10 +402,10 @@ class AutoCreateSchemaExecutor {
}
if (!templateExtendInfoMap.isEmpty()) {
- for (TemplateExtendInfo templateExtendInfo : templateExtendInfoMap.values()) {
- templateExtendInfo = templateExtendInfo.deduplicate();
+ for (Map.Entry<String, TemplateExtendInfo> entry : templateExtendInfoMap.entrySet()) {
+ TemplateExtendInfo templateExtendInfo = entry.getValue().deduplicate();
internalExtendTemplate(
- templateExtendInfo.getTemplateName(),
+ entry.getKey(),
templateExtendInfo.getMeasurements(),
templateExtendInfo.getDataTypes(),
templateExtendInfo.getEncodings(),