You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2025/02/20 01:57:06 UTC
(iotdb) branch ycy/planCacheDemo created (now 1cc6f5aed04)
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a change to branch ycy/planCacheDemo
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 1cc6f5aed04 support statement generalization
This branch includes the following new commits:
new 1cc6f5aed04 support statement generalization
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: support statement generalization
Posted by yc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/planCacheDemo
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1cc6f5aed042a20a21d118dfbaf626107af7bfbf
Author: ycycse <yc...@gmail.com>
AuthorDate: Thu Feb 20 09:56:48 2025 +0800
support statement generalization
---
.../relational/planner/TableLogicalPlanner.java | 27 ++-
.../optimizations/LogicalOptimizeFactory.java | 4 +-
.../plan/relational/sql/ast/Literal.java | 15 ++
.../relational/sql/ast/LiteralMarkerReplacer.java | 190 +++++++++++++++++++++
.../relational/sql/util/ExpressionFormatter.java | 26 +++
.../plan/relational/planner/PlanCacheTest.java | 57 +++++++
6 files changed, 316 insertions(+), 3 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
index 7ffc7d9ca14..ebfcf0268f5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
@@ -59,6 +59,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LiteralMarkerReplacer;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query;
@@ -75,6 +76,8 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.type.LongType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TypeFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -96,6 +99,7 @@ public class TableLogicalPlanner {
private final List<PlanOptimizer> planOptimizers;
private final Metadata metadata;
private final WarningCollector warningCollector;
+ private final Logger logger = LoggerFactory.getLogger(TableLogicalPlanner.class);
@TestOnly
public TableLogicalPlanner(
@@ -129,9 +133,30 @@ public class TableLogicalPlanner {
this.planOptimizers = planOptimizers;
}
+ private void generalizeStatement(Query query) {
+ LiteralMarkerReplacer literalMarkerReplacer = new LiteralMarkerReplacer();
+ literalMarkerReplacer.process(query);
+ }
+
public LogicalQueryPlan plan(final Analysis analysis) {
long startTime = System.nanoTime();
- final Statement statement = analysis.getStatement();
+ Statement statement = analysis.getStatement();
+
+ // Try to use plan cache
+ // We should check if statement is Query in enablePlanCache() method
+ // if(analysis.enablePlanCache()){
+ // generalizeStatement((Query) statement);
+ //
+ // String cachedKey = calculateCacheKey(generalizedStatement, analysis);
+ // PlanNode cachedPlan = queryContext.getPlanCache().get(cachedKey);
+ // if (cachedPlan != null) {
+ // // deal with the device stuff
+ // return new LogicalQueryPlan(queryContext, cachedPlan);
+ // }
+ // // Following implementation of plan should be based on the generalizedStatement
+ // statement = generalizedStatement;
+ // }
+
PlanNode planNode = planStatement(analysis, statement);
if (analysis.isQuery()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
index 97cd1401a55..b461c115aa5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
@@ -236,7 +236,7 @@ public class LogicalOptimizeFactory {
new IterativeOptimizer(
plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())),
simplifyOptimizer,
- new PushPredicateIntoTableScan(plannerContext, typeAnalyzer),
+ new PushPredicateIntoTableScan(plannerContext, typeAnalyzer), // *
// Currently, Distinct is not supported, so we cant use this rule for now.
// new IterativeOptimizer(
// plannerContext,
@@ -248,7 +248,7 @@ public class LogicalOptimizeFactory {
new IterativeOptimizer(plannerContext, ruleStats, limitPushdownRules),
new PushLimitOffsetIntoTableScan(),
new TransformAggregationToStreamable(),
- new PushAggregationIntoTableScan(),
+ new PushAggregationIntoTableScan(), // *
new TransformSortToStreamSort(),
new IterativeOptimizer(
plannerContext,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
index b60e5a454a7..6286f9c978b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
@@ -44,4 +44,19 @@ public abstract class Literal extends Expression {
* @return a value that can be processed by IoTDB storage engine
*/
public abstract Object getTsValue();
+
+ // For literalMarker
+ private int literalIndex = -1;
+
+ public void setLiteralIndex(int literalIndex) {
+ this.literalIndex = literalIndex;
+ }
+
+ public int getLiteralIndex() {
+ return literalIndex;
+ }
+
+ public boolean isLiteralMarker() {
+ return literalIndex != -1;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java
new file mode 100644
index 00000000000..c4ad3670935
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class LiteralMarkerReplacer extends AstVisitor<Void, Void> {
+
+ private static final String NOT_SUPPORTED =
+ "visit() not supported for %s in LiteralMarkerReplacer.";
+
+ List<Literal> literalList = new LinkedList<Literal>();
+ int literalGlobalIndex = -1;
+
+ public List<Literal> getLiteralList() {
+ return literalList;
+ }
+
+ @Override
+ protected Void visitNode(Node node, Void context) {
+ throw new UnsupportedOperationException(
+ "visitNode of Node {" + node.getClass() + "} is not supported in LiteralMarkerReplacer");
+ }
+
+ @Override
+ protected Void visitQuery(Query node, Void context) {
+ node.getQueryBody().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitQuerySpecification(QuerySpecification node, Void context) {
+ node.getSelect().accept(this, context);
+ node.getWhere().ifPresent(where -> where.accept(this, context));
+ node.getGroupBy().ifPresent(groupBy -> groupBy.accept(this, context));
+ node.getHaving().ifPresent(having -> having.accept(this, context));
+ node.getOrderBy().ifPresent(orderBy -> orderBy.accept(this, context));
+ node.getLimit().ifPresent(limit -> limit.accept(this, context));
+ node.getOffset().ifPresent(offset -> offset.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitSelect(Select node, Void context) {
+ node.getSelectItems().forEach(item -> item.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitAllColumns(AllColumns node, Void context) {
+ node.getTarget().ifPresent(target -> target.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitSingleColumn(SingleColumn node, Void context) {
+ node.getExpression().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitOrderBy(OrderBy node, Void context) {
+ node.getSortItems().forEach(item -> item.getSortKey().accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitGroupBy(GroupBy node, Void context) {
+ for (GroupingElement element : node.getGroupingElements()) {
+ element.getExpressions().forEach(expression -> expression.accept(this, context));
+ }
+ return context;
+ }
+
+ @Override
+ protected Void visitLimit(Limit node, Void context) {
+ node.getRowCount().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitOffset(Offset node, Void context) {
+ node.getRowCount().accept(this, context);
+ return context;
+ }
+
+ // ============================ Expression ============================
+
+ @Override
+ protected Void visitExpression(Expression node, Void context) {
+ // do nothing
+ return context;
+ }
+
+ @Override
+ protected Void visitLogicalExpression(LogicalExpression node, Void context) {
+ node.getTerms().forEach(term -> term.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitComparisonExpression(ComparisonExpression node, Void context) {
+ node.getLeft().accept(this, context);
+ node.getRight().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitIsNullPredicate(IsNullPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitIsNotNullPredicate(IsNotNullPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitFunctionCall(FunctionCall node, Void context) {
+ node.getArguments().forEach(arg -> arg.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitLikePredicate(LikePredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitBetweenPredicate(BetweenPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ node.getMin().accept(this, context);
+ node.getMax().accept(this, context);
+ return context;
+ }
+
+ // TODO it can be optimized by an array of expressions
+ @Override
+ protected Void visitInPredicate(InPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitNotExpression(NotExpression node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitArithmeticBinary(ArithmeticBinaryExpression node, Void context) {
+ node.getLeft().accept(this, context);
+ node.getRight().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitArithmeticUnary(ArithmeticUnaryExpression node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitLiteral(Literal node, Void context) {
+ literalList.add(node);
+ node.setLiteralIndex(++literalGlobalIndex);
+ return context;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
index d6222564ec7..e16b3e14c93 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
@@ -101,6 +101,8 @@ public final class ExpressionFormatter {
() ->
new DecimalFormat("0.###################E0###", new DecimalFormatSymbols(Locale.US)));
+ private static final String LITERAL_MARKER_FORMAT = "LiteralMarker(#%s)";
+
private ExpressionFormatter() {}
public static String formatExpression(Expression expression) {
@@ -178,6 +180,9 @@ public final class ExpressionFormatter {
@Override
protected String visitBooleanLiteral(BooleanLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> String.valueOf(node.getValue()));
@@ -185,6 +190,9 @@ public final class ExpressionFormatter {
@Override
protected String visitStringLiteral(StringLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> formatStringLiteral(node.getValue()));
@@ -192,6 +200,9 @@ public final class ExpressionFormatter {
@Override
protected String visitBinaryLiteral(BinaryLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> "X'" + node.toHexString() + "'");
@@ -209,11 +220,17 @@ public final class ExpressionFormatter {
@Override
protected String visitLongLiteral(LongLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter.map(formatter -> formatter.apply(node)).orElseGet(node::getValue);
}
@Override
protected String visitDoubleLiteral(DoubleLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> doubleFormatter.get().format(node.getValue()));
@@ -221,6 +238,9 @@ public final class ExpressionFormatter {
@Override
protected String visitDecimalLiteral(DecimalLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
// TODO return node value without "DECIMAL '..'" when
@@ -230,6 +250,9 @@ public final class ExpressionFormatter {
@Override
protected String visitGenericLiteral(GenericLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> node.getType() + " " + formatStringLiteral(node.getValue()));
@@ -237,6 +260,9 @@ public final class ExpressionFormatter {
@Override
protected String visitNullLiteral(NullLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter.map(formatter -> formatter.apply(node)).orElse("null");
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java
new file mode 100644
index 00000000000..4e160814b2e
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.planner;
+
+import org.apache.iotdb.db.protocol.session.IClientSession;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LiteralMarkerReplacer;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.util.SqlFormatter;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.time.ZoneId;
+
+public class PlanCacheTest {
+
+ @Test
+ public void statementGeneralizationTest() {
+
+ String databaseName = "tetsdb";
+ IClientSession clientSession = Mockito.mock(IClientSession.class);
+ Mockito.when(clientSession.getDatabaseName()).thenReturn(databaseName);
+
+ SqlParser sqlParser = new SqlParser();
+ String sql = "select id + 1 from table1 where id > 10 and time=13289078 order by rank+1";
+ long startTime = System.nanoTime();
+ Statement originalStatement =
+ sqlParser.createStatement(sql, ZoneId.systemDefault(), clientSession);
+ System.out.println("Time to parse: " + (System.nanoTime() - startTime));
+
+ LiteralMarkerReplacer literalMarkerReplacer = new LiteralMarkerReplacer();
+ startTime = System.nanoTime();
+ literalMarkerReplacer.process(originalStatement);
+
+ String newSql = SqlFormatter.formatSql(originalStatement);
+ System.out.println("Time to replace: " + (System.nanoTime() - startTime));
+ System.out.println(newSql);
+ }
+}