You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/07/15 02:23:06 UTC
(iotdb) 01/02: Fixed the issue that non aligned tree view query used too much memory
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 963532caeaba63fdfec3ac3d94cce0894fdeff15
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Tue Jul 15 08:47:13 2025 +0800
Fixed the issue that non aligned tree view query used too much memory
(cherry picked from commit d7844931a87a9478c5ada93276202e13565cae22)
---
.../plan/planner/TableOperatorGenerator.java | 4 +-
...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 ++++++++++++++--------
2 files changed, 46 insertions(+), 27 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 17f53216d40..e19204e2480 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -648,9 +648,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder();
// time filter may be stateful, so we need to copy it
builder.withGlobalTimeFilter(timeFilter == null ? null : timeFilter.copy());
- builder
- .withIsTableViewForTreeModel(true)
- .withAllSensors(new HashSet<>(measurementColumnNames));
+ builder.withIsTableViewForTreeModel(true).withAllSensors(allSensors);
if (pushDownPredicateForCurrentMeasurement != null) {
builder.withPushDownFilter(
convertPredicateToFilter(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
index 2188a04479b..01a00529503 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
@@ -126,28 +126,20 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
new Symbol("time"),
new ColumnSchema(
"time", TypeFactory.getType(TSDataType.INT64), false, TsTableColumnCategory.TIME));
- columnSchemaMap.put(
- new Symbol("sensor0"),
- new ColumnSchema(
- "sensor0", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor1"),
- new ColumnSchema(
- "sensor1", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor2"),
- new ColumnSchema(
- "sensor2", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor3"),
- new ColumnSchema(
- "sensor3", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
+ for (int i = 0; i < 10000; i++) {
+ columnSchemaMap.put(
+ new Symbol("sensor" + i),
+ new ColumnSchema(
+ "sensor" + i,
+ TypeFactory.getType(TSDataType.INT32),
+ false,
+ TsTableColumnCategory.FIELD));
+ }
Map<Symbol, Type> symbolTSDataTypeMap = new HashMap<>();
- symbolTSDataTypeMap.put(new Symbol("sensor0"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor1"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor2"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor3"), TypeFactory.getType(TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ symbolTSDataTypeMap.put(new Symbol("sensor" + i), TypeFactory.getType(TSDataType.INT32));
+ }
symbolTSDataTypeMap.put(new Symbol("time"), TypeFactory.getType(TypeEnum.INT64));
symbolTSDataTypeMap.put(new Symbol("tag1"), TypeFactory.getType(TSDataType.TEXT));
typeProvider = new TypeProvider(symbolTSDataTypeMap);
@@ -158,10 +150,9 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING));
tsTable.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64));
tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.TEXT));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor0", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor1", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor2", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor3", TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ tsTable.addColumnSchema(new FieldColumnSchema("sensor" + i, TSDataType.INT32));
+ }
tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + "");
tsTable.addProp(
TreeViewSchema.TREE_PATH_PATTERN,
@@ -176,6 +167,36 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
SeriesReaderTestUtil.tearDown(seqResources, unSeqResources);
}
+ @Test
+ public void testQueryManyDevices() throws Exception {
+ List<String> outputColumnList = new ArrayList<>(10000 + 2);
+ TreeNonAlignedDeviceViewScanNode node = getTreeNonAlignedDeviceViewScanNode(outputColumnList);
+ node.setPushDownOffset(500);
+ node.setPushDownLimit(500);
+ node.setPushDownPredicate(
+ new ComparisonExpression(
+ ComparisonExpression.Operator.GREATER_THAN,
+ new Symbol("sensor1").toSymbolReference(),
+ new LongLiteral("1000")));
+ for (int i = 0; i < 10000; i++) {
+ outputColumnList.add("sensor" + i);
+ }
+ outputColumnList.add("time");
+ outputColumnList.add("tag1");
+ ExecutorService instanceNotificationExecutor =
+ IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification");
+ Operator operator = getOperator(node, instanceNotificationExecutor);
+ try {
+ assertTrue(operator instanceof DeviceIteratorScanOperator);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ operator.close();
+ instanceNotificationExecutor.shutdown();
+ }
+ }
+
@Test
public void testScanWithPushDownPredicateAndLimitAndOffset() throws Exception {
List<String> outputColumnList = Arrays.asList("sensor0", "sensor1", "sensor2", "time", "tag1");