You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2024/06/10 14:50:39 UTC
(iotdb) 03/03: cp from new-table-model-debug
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch TableModelGrammar0606
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ed474142d3e4edef52f6237f8915e71ee6cfeed3
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu Jun 6 22:44:44 2024 +0800
cp from new-table-model-debug
---
.../db/queryengine/common/MPPQueryContext.java | 4 +
.../relational/ColumnTransformerBuilder.java | 2 +-
....java => PredicatePushIntoMetadataChecker.java} | 8 +-
.../planner/optimizations/IndexScan.java | 49 ++++++++--
.../planner/optimizations/PredicatePushDown.java | 105 ++++++++++++++++++++-
.../plan/optimization/PredicatePushDownTest.java | 2 +-
6 files changed, 154 insertions(+), 16 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
index 988224da729..bafd57aa3db 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider;
import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics;
import org.apache.tsfile.read.filter.basic.Filter;
@@ -89,6 +90,9 @@ public class MPPQueryContext {
private final LocalExecutionPlanner LOCAL_EXECUTION_PLANNER = LocalExecutionPlanner.getInstance();
+ // 0 : metadataExpressions, 1 : expressionsPushDownToOperator, 2 : expressionsCannotPushDown
+ public List<List<Expression>> splitPredicateExpression;
+
public MPPQueryContext(QueryId queryId) {
this.queryId = queryId;
this.endPointBlackList = new LinkedList<>();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
index fb405d1c62e..b911e2f0da4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
@@ -113,7 +113,7 @@ import java.util.Optional;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
-import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoIndexScanChecker.isStringLiteral;
+import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker.isStringLiteral;
import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType;
import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature;
import static org.apache.tsfile.read.common.type.BinaryType.TEXT;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java
similarity index 93%
rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java
rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java
index f7d2c1764a3..4561b6a838f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java
@@ -40,11 +40,15 @@ import java.util.Set;
import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference;
-public class PredicatePushIntoIndexScanChecker extends PredicateVisitor<Boolean, Void> {
+public class PredicatePushIntoMetadataChecker extends PredicateVisitor<Boolean, Void> {
private final Set<String> idOrAttributeColumnNames;
- public PredicatePushIntoIndexScanChecker(Set<String> idOrAttributeColumnNames) {
+ public static boolean check(Set<String> idOrAttributeColumnNames, Expression expression) {
+ return new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames).process(expression);
+ }
+
+ public PredicatePushIntoMetadataChecker(Set<String> idOrAttributeColumnNames) {
this.idOrAttributeColumnNames = idOrAttributeColumnNames;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
index fd3fe00bd9f..db728f9ae52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
-import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoIndexScanChecker;
+import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName;
@@ -85,20 +85,28 @@ public class IndexScan implements RelationalPlanOptimizer {
@Override
public PlanNode visitFilter(FilterNode node, RewriterContext context) {
context.setPredicate(node.getPredicate());
+ context.setFilterNode(node);
node.getChild().accept(this, context);
return node;
}
@Override
public PlanNode visitTableScan(TableScanNode node, RewriterContext context) {
- List<String> attributeColumns =
- node.getAssignments().entrySet().stream()
- .filter(e -> e.getValue().getColumnCategory().equals(ATTRIBUTE))
- .map(e -> e.getKey().getName())
- .collect(Collectors.toList());
- List<Expression> conjExpressions = getConjunctionExpressions(context.getPredicate(), node);
+ // only when exist diff predicate in FilterNode, context.predicate will not equal null
+ if (context.predicate == null) {
+ context.predicate = node.getPushDownPredicate();
+ }
+
+ List<Expression> conjExpressions = getConjunctionExpressions(context, node);
+ node.setPushDownPredicate(context.getPredicate());
String dbName = context.getSessionInfo().getDatabaseName().get();
+ List<String> attributeColumns =
+ node.getOutputSymbols().stream()
+ .filter(
+ symbol -> ATTRIBUTE.equals(node.getAssignments().get(symbol).getColumnCategory()))
+ .map(Symbol::getName)
+ .collect(Collectors.toList());
List<DeviceEntry> deviceEntries =
context
.getMetadata()
@@ -152,7 +160,8 @@ public class IndexScan implements RelationalPlanOptimizer {
}
private static List<Expression> getConjunctionExpressions(
- Expression predicate, TableScanNode node) {
+ RewriterContext context, TableScanNode node) {
+ Expression predicate = context.predicate;
if (predicate == null) {
return Collections.emptyList();
}
@@ -164,20 +173,31 @@ public class IndexScan implements RelationalPlanOptimizer {
if (predicate instanceof LogicalExpression
&& ((LogicalExpression) predicate).getOperator() == LogicalExpression.Operator.AND) {
List<Expression> resultExpressions = new ArrayList<>();
+ List<Expression> beExpressions = new ArrayList<>();
for (Expression subExpression : ((LogicalExpression) predicate).getTerms()) {
if (Boolean.TRUE.equals(
- new PredicatePushIntoIndexScanChecker(idOrAttributeColumnNames)
+ new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames)
.process(subExpression))) {
resultExpressions.add(subExpression);
+ } else {
+ beExpressions.add(subExpression);
}
}
+ if (beExpressions.isEmpty()) {
+ context.predicate = null;
+ } else if (beExpressions.size() == 1) {
+ context.predicate = beExpressions.get(0);
+ } else {
+ context.predicate = new LogicalExpression(LogicalExpression.Operator.AND, beExpressions);
+ }
return resultExpressions;
}
if (Boolean.FALSE.equals(
- new PredicatePushIntoIndexScanChecker(idOrAttributeColumnNames).process(predicate))) {
+ new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames).process(predicate))) {
return Collections.emptyList();
} else {
+ context.predicate = null;
return Collections.singletonList(predicate);
}
}
@@ -219,6 +239,7 @@ public class IndexScan implements RelationalPlanOptimizer {
private final Analysis analysis;
private final IPartitionFetcher partitionFetcher;
private final MPPQueryContext queryContext;
+ private FilterNode filterNode;
RewriterContext(
Expression predicate,
@@ -266,5 +287,13 @@ public class IndexScan implements RelationalPlanOptimizer {
public MPPQueryContext getQueryContext() {
return queryContext;
}
+
+ public FilterNode getFilterNode() {
+ return filterNode;
+ }
+
+ public void setFilterNode(FilterNode filterNode) {
+ this.filterNode = filterNode;
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
index ae05611ce08..842448be624 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
@@ -22,15 +22,23 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
+import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.FunctionCall;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.IsNullPredicate;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.LogicalExpression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Node;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.NotExpression;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
/** Push down predicate to TableScanNode as possible. */
public class PredicatePushDown implements RelationalPlanOptimizer {
@@ -78,9 +86,12 @@ public class PredicatePushDown implements RelationalPlanOptimizer {
@Override
public PlanNode visitFilter(FilterNode node, RewriterContext context) {
+ context.filterNode = node;
+
if (node.getPredicate() != null) {
// when exist diff function, predicate can not be pushed down
if (containsDiffFunction(node.getPredicate())) {
+ context.pushDownPredicate = null;
return node;
}
@@ -91,17 +102,85 @@ public class PredicatePushDown implements RelationalPlanOptimizer {
return node.getChild();
}
+ // when reach this case?
node.getChild().accept(this, context);
- return node;
+ return node.getChild();
}
@Override
public PlanNode visitTableScan(TableScanNode node, RewriterContext context) {
- node.setPushDownPredicate(context.pushDownPredicate);
+ // has diff in FilterNode
+ if (context.pushDownPredicate == null) {
+ node.setPushDownPredicate(null);
+ return node;
+ }
+
+ context.queryContext.splitPredicateExpression = splitConjunctionExpressions(context, node);
+
+ if (context.queryContext.splitPredicateExpression.get(1).size() > 0) {
+ List<Expression> expressions = context.queryContext.splitPredicateExpression.get(1);
+ node.setPushDownPredicate(
+ expressions.size() == 1
+ ? expressions.get(0)
+ : new LogicalExpression(LogicalExpression.Operator.AND, expressions));
+ } else {
+ node.setPushDownPredicate(null);
+ }
+
+ // exist expressions can not push down
+ if (context.queryContext.splitPredicateExpression.get(2).size() > 0) {
+ List<Expression> expressions = context.queryContext.splitPredicateExpression.get(2);
+ return new FilterNode(
+ context.queryContext.getQueryId().genPlanNodeId(),
+ node,
+ expressions.size() == 1
+ ? expressions.get(0)
+ : new LogicalExpression(LogicalExpression.Operator.AND, expressions));
+ }
+
return node;
}
}
+ private static List<List<Expression>> splitConjunctionExpressions(
+ RewriterContext context, TableScanNode node) {
+ Expression predicate = context.pushDownPredicate;
+
+ Set<String> idOrAttributeColumnNames =
+ node.getIdAndAttributeIndexMap().keySet().stream()
+ .map(Symbol::getName)
+ .collect(Collectors.toSet());
+
+ List<Expression> metadataExpressions = new ArrayList<>();
+ List<Expression> expressionsCanPushDownToOperator = new ArrayList<>();
+ List<Expression> expressionsCannotPushDown = new ArrayList<>();
+
+ if (predicate instanceof LogicalExpression
+ && ((LogicalExpression) predicate).getOperator() == LogicalExpression.Operator.AND) {
+
+ for (Expression expression : ((LogicalExpression) predicate).getTerms()) {
+ if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, expression)) {
+ metadataExpressions.add(expression);
+ } else if (cannotPushDownToOperator(expression)) {
+ expressionsCannotPushDown.add(expression);
+ } else {
+ expressionsCanPushDownToOperator.add(expression);
+ }
+ }
+ }
+
+ if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, predicate)) {
+ metadataExpressions.add(predicate);
+ } else if (cannotPushDownToOperator(predicate)) {
+ expressionsCannotPushDown.add(predicate);
+ } else {
+ expressionsCanPushDownToOperator.add(predicate);
+ }
+
+ return Arrays.asList(
+ metadataExpressions, expressionsCanPushDownToOperator, expressionsCannotPushDown);
+ }
+
static boolean containsDiffFunction(Expression expression) {
if (expression instanceof FunctionCall
&& "diff".equalsIgnoreCase(((FunctionCall) expression).getName().toString())) {
@@ -119,7 +198,29 @@ public class PredicatePushDown implements RelationalPlanOptimizer {
return false;
}
+ /**
+ * When filter exists NotExpression or IsNullPredicate, this filter can not be pushed down into
+ * Operator
+ */
+ static boolean cannotPushDownToOperator(Expression expression) {
+ if (expression instanceof NotExpression || expression instanceof IsNullPredicate) {
+ return true;
+ }
+
+ if (!expression.getChildren().isEmpty()) {
+ for (Node node : expression.getChildren()) {
+ if (containsDiffFunction((Expression) node)) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
private static class RewriterContext {
Expression pushDownPredicate;
+ MPPQueryContext queryContext;
+ FilterNode filterNode;
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java
index 9d67070dc47..dc6cc08d7dd 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java
@@ -295,7 +295,7 @@ public class PredicatePushDownTest {
}
@Test
- public void testCannotPushDownAlignByTime() {
+ public void testCannotPushDownToOperatorAlignByTime() {
checkCannotPushDown(
"select s1, s2 from root.sg.d1 where time > 100 and (s1 > 10 or s2 > 10)",
new TestPlanBuilder()