You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2025/07/14 11:41:37 UTC
(iotdb) 01/01: fix non-aligned tree view query oom
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch fixTableViewQueryOOM
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1f9497644dc0fbe8c401559fdfc8b8bde6a78640
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Mon Jul 14 19:40:05 2025 +0800
fix non-aligned tree view query oom
---
.../plan/planner/TableOperatorGenerator.java | 5 +-
...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 ++++++++++++++--------
2 files changed, 48 insertions(+), 26 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 132f5ebcc00..55708877dd5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -635,7 +635,8 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
pushDownOffsetAndLimitToLeftChildSeriesScanOperator
|| pushDownOffsetAndLimitAfterInnerJoinOperator
|| isSingleColumn;
- for (Symbol symbol : measurementSchemaIndex2Symbols) {
+ for (int i = 0; i < measurementSchemaIndex2Symbols.size(); i++) {
+ Symbol symbol = measurementSchemaIndex2Symbols.get(i);
List<Expression> pushDownPredicatesForCurrentMeasurement =
pushDownConjunctsForEachMeasurement.get(symbol);
Expression pushDownPredicateForCurrentMeasurement =
@@ -649,7 +650,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
builder.withGlobalTimeFilter(timeFilter == null ? null : timeFilter.copy());
builder
.withIsTableViewForTreeModel(true)
- .withAllSensors(new HashSet<>(measurementColumnNames));
+ .withAllSensors(Collections.singleton(measurementColumnNames.get(i)));
if (pushDownPredicateForCurrentMeasurement != null) {
builder.withPushDownFilter(
convertPredicateToFilter(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
index 2188a04479b..01a00529503 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java
@@ -126,28 +126,20 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
new Symbol("time"),
new ColumnSchema(
"time", TypeFactory.getType(TSDataType.INT64), false, TsTableColumnCategory.TIME));
- columnSchemaMap.put(
- new Symbol("sensor0"),
- new ColumnSchema(
- "sensor0", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor1"),
- new ColumnSchema(
- "sensor1", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor2"),
- new ColumnSchema(
- "sensor2", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
- columnSchemaMap.put(
- new Symbol("sensor3"),
- new ColumnSchema(
- "sensor3", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD));
+ for (int i = 0; i < 10000; i++) {
+ columnSchemaMap.put(
+ new Symbol("sensor" + i),
+ new ColumnSchema(
+ "sensor" + i,
+ TypeFactory.getType(TSDataType.INT32),
+ false,
+ TsTableColumnCategory.FIELD));
+ }
Map<Symbol, Type> symbolTSDataTypeMap = new HashMap<>();
- symbolTSDataTypeMap.put(new Symbol("sensor0"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor1"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor2"), TypeFactory.getType(TSDataType.INT32));
- symbolTSDataTypeMap.put(new Symbol("sensor3"), TypeFactory.getType(TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ symbolTSDataTypeMap.put(new Symbol("sensor" + i), TypeFactory.getType(TSDataType.INT32));
+ }
symbolTSDataTypeMap.put(new Symbol("time"), TypeFactory.getType(TypeEnum.INT64));
symbolTSDataTypeMap.put(new Symbol("tag1"), TypeFactory.getType(TSDataType.TEXT));
typeProvider = new TypeProvider(symbolTSDataTypeMap);
@@ -158,10 +150,9 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING));
tsTable.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64));
tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.TEXT));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor0", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor1", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor2", TSDataType.INT32));
- tsTable.addColumnSchema(new FieldColumnSchema("sensor3", TSDataType.INT32));
+ for (int i = 0; i < 10000; i++) {
+ tsTable.addColumnSchema(new FieldColumnSchema("sensor" + i, TSDataType.INT32));
+ }
tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + "");
tsTable.addProp(
TreeViewSchema.TREE_PATH_PATTERN,
@@ -176,6 +167,36 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest {
SeriesReaderTestUtil.tearDown(seqResources, unSeqResources);
}
+ @Test
+ public void testQueryManyDevices() throws Exception {
+ List<String> outputColumnList = new ArrayList<>(10000 + 2);
+ TreeNonAlignedDeviceViewScanNode node = getTreeNonAlignedDeviceViewScanNode(outputColumnList);
+ node.setPushDownOffset(500);
+ node.setPushDownLimit(500);
+ node.setPushDownPredicate(
+ new ComparisonExpression(
+ ComparisonExpression.Operator.GREATER_THAN,
+ new Symbol("sensor1").toSymbolReference(),
+ new LongLiteral("1000")));
+ for (int i = 0; i < 10000; i++) {
+ outputColumnList.add("sensor" + i);
+ }
+ outputColumnList.add("time");
+ outputColumnList.add("tag1");
+ ExecutorService instanceNotificationExecutor =
+ IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification");
+ Operator operator = getOperator(node, instanceNotificationExecutor);
+ try {
+ assertTrue(operator instanceof DeviceIteratorScanOperator);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ operator.close();
+ instanceNotificationExecutor.shutdown();
+ }
+ }
+
@Test
public void testScanWithPushDownPredicateAndLimitAndOffset() throws Exception {
List<String> outputColumnList = Arrays.asList("sensor0", "sensor1", "sensor2", "time", "tag1");