You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/09/03 08:06:37 UTC
(iotdb) branch ty/TryOpt created (now d008c227648)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/TryOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at d008c227648 PruneColumn firstly in RelationPlanner
This branch includes the following new commits:
new d008c227648 PruneColumn firstly in RelationPlanner
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: PruneColumn firstly in RelationPlanner
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/TryOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d008c227648f9e170ab6d56011ae770a6632e7ae
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Sep 3 16:06:10 2024 +0800
PruneColumn firstly in RelationPlanner
---
.../plan/relational/analyzer/Analysis.java | 24 ++++++
.../relational/analyzer/StatementAnalyzer.java | 23 ++++--
.../plan/relational/planner/RelationPlanner.java | 29 ++++---
.../optimizations/LogicalOptimizeFactory.java | 89 ++++++++++++++++++++--
4 files changed, 140 insertions(+), 25 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
index 87e7fe15548..6250f394cf4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
@@ -70,6 +70,7 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -479,6 +480,9 @@ public class Analysis implements IAnalysis {
}
private <T extends Node> List<T> dereference(Collection<NodeRef<T>> nodeRefs) {
+ if (nodeRefs.isEmpty()) {
+ return Collections.emptyList();
+ }
return nodeRefs.stream().map(NodeRef::getNode).collect(toImmutableList());
}
@@ -517,6 +521,26 @@ public class Analysis implements IAnalysis {
return columnReferences.containsKey(NodeRef.of(expression));
}
+ public boolean hasFieldReference(QualifiedObjectName tableName, String fieldName) {
+ for (Map<QualifiedObjectName, Set<String>> map : tableColumnReferences.values()) {
+ Set<String> fields = map.get(tableName);
+ if (fields != null && fields.contains(fieldName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public Set<String> getUsedColumns(QualifiedObjectName tableName) {
+ for (Map<QualifiedObjectName, Set<String>> map : tableColumnReferences.values()) {
+ Set<String> fields = map.get(tableName);
+ if (fields != null) {
+ return fields;
+ }
+ }
+ return Collections.emptySet();
+ }
+
public void addTableColumnReferences(
AccessControl accessControl,
Identity identity,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index a2b91ac794d..df2ee265edf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -1381,10 +1381,18 @@ public class StatementAnalyzer {
}
if (name != null) {
- List<Field> matchingFields = sourceScope.getRelationType().resolveFields(name);
- if (!matchingFields.isEmpty()) {
- originTable = matchingFields.get(0).getOriginTable();
- originColumn = matchingFields.get(0).getOriginColumnName();
+ Field matchingField = null;
+ try {
+ matchingField = analysis.getResolvedField(expression).getField();
+ } catch (IllegalArgumentException e) {
+ List<Field> matchingFields = sourceScope.getRelationType().resolveFields(name);
+ if (!matchingFields.isEmpty()) {
+ matchingField = matchingFields.get(0);
+ }
+ }
+ if (matchingField != null) {
+ originTable = matchingField.getOriginTable();
+ originColumn = matchingField.getOriginColumnName();
}
}
@@ -1582,15 +1590,14 @@ public class StatementAnalyzer {
List<Field> outputFields = fields.build();
+ RelationType relationType = new RelationType(outputFields);
Scope accessControlScope =
- Scope.builder()
- .withRelationType(RelationId.anonymous(), new RelationType(outputFields))
- .build();
+ Scope.builder().withRelationType(RelationId.anonymous(), relationType).build();
// analyzeFiltersAndMasks(table, name, new RelationType(outputFields),
// accessControlScope);
analysis.registerTable(table, tableSchema, name);
- Scope tableScope = createAndAssignScope(table, scope, outputFields);
+ Scope tableScope = createAndAssignScope(table, scope, relationType);
// if (addRowIdColumn) {
// FieldReference reference = new FieldReference(outputFields.size() - 1);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index cdfb2023ae9..5a112f3c911 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -60,6 +60,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
@@ -128,14 +129,32 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder();
ImmutableMap.Builder<Symbol, ColumnSchema> symbolToColumnSchema = ImmutableMap.builder();
Collection<Field> fields = scope.getRelationType().getAllFields();
+ QualifiedName qualifiedName = analysis.getRelationName(table);
+ if (!qualifiedName.getPrefix().isPresent()) {
+ throw new IllegalStateException("Table " + table.getName() + " has no prefix!");
+ }
+
+ QualifiedObjectName qualifiedObjectName =
+ new QualifiedObjectName(
+ qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null),
+ qualifiedName.getSuffix());
+
+ Set<String> usedColumns = analysis.getUsedColumns(qualifiedObjectName);
+
// on the basis of that the order of fields is same with the column category order of segments
// in DeviceEntry
Map<Symbol, Integer> idAndAttributeIndexMap = new HashMap<>();
int idIndex = 0;
for (Field field : fields) {
+ TsTableColumnCategory category = field.getColumnCategory();
+ // only keep used columns and all ID columns
+ if (category != TsTableColumnCategory.ID
+ && field.getOriginColumnName().isPresent()
+ && usedColumns.contains(field.getOriginColumnName().get())) {
+ continue;
+ }
Symbol symbol = symbolAllocator.newSymbol(field);
outputSymbolsBuilder.add(symbol);
- TsTableColumnCategory category = field.getColumnCategory();
symbolToColumnSchema.put(
symbol,
new ColumnSchema(
@@ -146,15 +165,7 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
}
List<Symbol> outputSymbols = outputSymbolsBuilder.build();
- QualifiedName qualifiedName = analysis.getRelationName(table);
- if (!qualifiedName.getPrefix().isPresent()) {
- throw new IllegalStateException("Table " + table.getName() + " has no prefix!");
- }
- QualifiedObjectName qualifiedObjectName =
- new QualifiedObjectName(
- qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null),
- qualifiedName.getSuffix());
Map<Symbol, ColumnSchema> tableColumnSchema = symbolToColumnSchema.build();
analysis.addTableSchema(qualifiedObjectName, tableColumnSchema);
TableScanNode tableScanNode =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
index 8acede85ba5..415fa40d6a6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
@@ -53,6 +53,7 @@ public class LogicalOptimizeFactory {
public LogicalOptimizeFactory(PlannerContext plannerContext) {
IrTypeAnalyzer typeAnalyzer = new IrTypeAnalyzer(plannerContext);
Metadata metadata = plannerContext.getMetadata();
+ RuleStatsRecorder ruleStats = new RuleStatsRecorder();
PlanOptimizer pushPredicateIntoTableScanOptimizer = new PushPredicateIntoTableScan();
PlanOptimizer transformSortToStreamSortOptimizer = new TransformSortToStreamSort();
@@ -68,12 +69,31 @@ public class LogicalOptimizeFactory {
new PruneTableScanColumns(plannerContext.getMetadata()),
new PruneTopKColumns());
IterativeOptimizer columnPruningOptimizer =
- new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), columnPruningRules);
+ new IterativeOptimizer(plannerContext, ruleStats, columnPruningRules);
+
+ // Set<Rule<?>> projectionPushdownRules = ImmutableSet.of(
+ // new PushProjectionThroughUnion(),
+ // new PushProjectionThroughExchange(),
+ // // Dereference pushdown rules
+ // new PushDownDereferencesThroughMarkDistinct(typeAnalyzer),
+ // new PushDownDereferenceThroughProject(typeAnalyzer),
+ // new PushDownDereferenceThroughUnnest(typeAnalyzer),
+ // new PushDownDereferenceThroughSemiJoin(typeAnalyzer),
+ // new PushDownDereferenceThroughJoin(typeAnalyzer),
+ // new PushDownDereferenceThroughFilter(typeAnalyzer),
+ // new ExtractDereferencesFromFilterAboveScan(typeAnalyzer),
+ // new PushDownDereferencesThroughLimit(typeAnalyzer),
+ // new PushDownDereferencesThroughSort(typeAnalyzer),
+ // new PushDownDereferencesThroughAssignUniqueId(typeAnalyzer),
+ // new PushDownDereferencesThroughWindow(typeAnalyzer),
+ // new PushDownDereferencesThroughTopN(typeAnalyzer),
+ // new PushDownDereferencesThroughRowNumber(typeAnalyzer),
+ // new PushDownDereferencesThroughTopNRanking(typeAnalyzer));
IterativeOptimizer inlineProjectionLimitFiltersOptimizer =
new IterativeOptimizer(
plannerContext,
- new RuleStatsRecorder(),
+ ruleStats,
ImmutableSet.of(
new InlineProjections(plannerContext),
new RemoveRedundantIdentityProjections(),
@@ -88,24 +108,77 @@ public class LogicalOptimizeFactory {
.add(new RemoveTrivialFilters())
.build();
IterativeOptimizer simplifyOptimizer =
- new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), simplifyOptimizerRules);
+ new IterativeOptimizer(plannerContext, ruleStats, simplifyOptimizerRules);
+ Set<Rule<?>> limitPushdownRules =
+ ImmutableSet.of(new PushLimitThroughOffset(), new PushLimitThroughProject());
IterativeOptimizer limitPushdownOptimizer =
- new IterativeOptimizer(
- plannerContext,
- new RuleStatsRecorder(),
- ImmutableSet.of(new PushLimitThroughOffset(), new PushLimitThroughProject()));
+ new IterativeOptimizer(plannerContext, ruleStats, limitPushdownRules);
PlanOptimizer pushLimitOffsetIntoTableScanOptimizer = new PushLimitOffsetIntoTableScan();
IterativeOptimizer topKOptimizer =
new IterativeOptimizer(
plannerContext,
- new RuleStatsRecorder(),
+ ruleStats,
ImmutableSet.of(new MergeLimitWithSort(), new MergeLimitOverProjectWithSort()));
this.planOptimizers =
ImmutableList.of(
+ new IterativeOptimizer(
+ plannerContext,
+ ruleStats,
+ ImmutableSet.<Rule<?>>builder()
+ .addAll(columnPruningRules)
+ // .addAll(projectionPushdownRules)
+ // .addAll(new UnwrapRowSubscript().rules())
+ // .addAll(new PushCastIntoRow().rules())
+ .addAll(
+ ImmutableSet.of(
+ new MergeFilters(),
+ new InlineProjections(plannerContext),
+ new RemoveRedundantIdentityProjections(),
+ new MergeLimits(),
+ new RemoveTrivialFilters()
+ // new RemoveRedundantLimit(),
+ // new RemoveRedundantOffset(),
+ // new RemoveRedundantSort(),
+ // new RemoveRedundantSortBelowLimitWithTies(),
+ // new RemoveRedundantTopN(),
+ // new RemoveRedundantDistinctLimit(),
+ // new ReplaceRedundantJoinWithSource(),
+ // new RemoveRedundantJoin(),
+ // new ReplaceRedundantJoinWithProject(),
+ // new RemoveRedundantExists(),
+ // new RemoveRedundantWindow(),
+ // new SingleDistinctAggregationToGroupBy(),
+ // new MergeLimitWithDistinct(),
+ // new PruneCountAggregationOverScalar(metadata),
+ // new SimplifyCountOverConstant(plannerContext),
+ // new
+ // PreAggregateCaseAggregations(plannerContext, typeAnalyzer)))
+ ))
+ .build()),
+ // MergeUnion and related projection pruning rules must run before limit pushdown rules,
+ // otherwise
+ // an intermediate limit node will prevent unions from being merged later on
+ new IterativeOptimizer(
+ plannerContext,
+ ruleStats,
+ ImmutableSet.<Rule<?>>builder()
+ // .addAll(projectionPushdownRules)
+ .addAll(columnPruningRules)
+ .addAll(limitPushdownRules)
+ .addAll(
+ ImmutableSet.of(
+ // new MergeUnion(),
+ // new RemoveEmptyUnionBranches(),
+ new MergeFilters(),
+ new RemoveTrivialFilters(),
+ new MergeLimits(),
+ new InlineProjections(plannerContext),
+ new RemoveRedundantIdentityProjections()))
+ .build()),
simplifyOptimizer,
columnPruningOptimizer,
inlineProjectionLimitFiltersOptimizer,