You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2025/02/20 01:57:07 UTC
(iotdb) 01/01: support statement generalization
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/planCacheDemo
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1cc6f5aed042a20a21d118dfbaf626107af7bfbf
Author: ycycse <yc...@gmail.com>
AuthorDate: Thu Feb 20 09:56:48 2025 +0800
support statement generalization
---
.../relational/planner/TableLogicalPlanner.java | 27 ++-
.../optimizations/LogicalOptimizeFactory.java | 4 +-
.../plan/relational/sql/ast/Literal.java | 15 ++
.../relational/sql/ast/LiteralMarkerReplacer.java | 190 +++++++++++++++++++++
.../relational/sql/util/ExpressionFormatter.java | 26 +++
.../plan/relational/planner/PlanCacheTest.java | 57 +++++++
6 files changed, 316 insertions(+), 3 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
index 7ffc7d9ca14..ebfcf0268f5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
@@ -59,6 +59,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LiteralMarkerReplacer;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query;
@@ -75,6 +76,8 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.type.LongType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TypeFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -96,6 +99,7 @@ public class TableLogicalPlanner {
private final List<PlanOptimizer> planOptimizers;
private final Metadata metadata;
private final WarningCollector warningCollector;
+ private final Logger logger = LoggerFactory.getLogger(TableLogicalPlanner.class);
@TestOnly
public TableLogicalPlanner(
@@ -129,9 +133,30 @@ public class TableLogicalPlanner {
this.planOptimizers = planOptimizers;
}
+ private void generalizeStatement(Query query) {
+ LiteralMarkerReplacer literalMarkerReplacer = new LiteralMarkerReplacer();
+ literalMarkerReplacer.process(query);
+ }
+
public LogicalQueryPlan plan(final Analysis analysis) {
long startTime = System.nanoTime();
- final Statement statement = analysis.getStatement();
+ Statement statement = analysis.getStatement();
+
+ // Try to use plan cache
+ // We should check if statement is Query in enablePlanCache() method
+ // if(analysis.enablePlanCache()){
+ // generalizeStatement((Query) statement);
+ //
+ // String cachedKey = calculateCacheKey(generalizedStatement, analysis);
+ // PlanNode cachedPlan = queryContext.getPlanCache().get(cachedKey);
+ // if (cachedPlan != null) {
+ // // deal with the device stuff
+ // return new LogicalQueryPlan(queryContext, cachedPlan);
+ // }
+ // // Following implementation of plan should be based on the generalizedStatement
+ // statement = generalizedStatement;
+ // }
+
PlanNode planNode = planStatement(analysis, statement);
if (analysis.isQuery()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
index 97cd1401a55..b461c115aa5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
@@ -236,7 +236,7 @@ public class LogicalOptimizeFactory {
new IterativeOptimizer(
plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())),
simplifyOptimizer,
- new PushPredicateIntoTableScan(plannerContext, typeAnalyzer),
+ new PushPredicateIntoTableScan(plannerContext, typeAnalyzer), // *
// Currently, Distinct is not supported, so we cant use this rule for now.
// new IterativeOptimizer(
// plannerContext,
@@ -248,7 +248,7 @@ public class LogicalOptimizeFactory {
new IterativeOptimizer(plannerContext, ruleStats, limitPushdownRules),
new PushLimitOffsetIntoTableScan(),
new TransformAggregationToStreamable(),
- new PushAggregationIntoTableScan(),
+ new PushAggregationIntoTableScan(), // *
new TransformSortToStreamSort(),
new IterativeOptimizer(
plannerContext,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
index b60e5a454a7..6286f9c978b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Literal.java
@@ -44,4 +44,19 @@ public abstract class Literal extends Expression {
* @return a value that can be processed by IoTDB storage engine
*/
public abstract Object getTsValue();
+
+ // For literalMarker
+ private int literalIndex = -1;
+
+ public void setLiteralIndex(int literalIndex) {
+ this.literalIndex = literalIndex;
+ }
+
+ public int getLiteralIndex() {
+ return literalIndex;
+ }
+
+ public boolean isLiteralMarker() {
+ return literalIndex != -1;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java
new file mode 100644
index 00000000000..c4ad3670935
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LiteralMarkerReplacer.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class LiteralMarkerReplacer extends AstVisitor<Void, Void> {
+
+ private static final String NOT_SUPPORTED =
+ "visit() not supported for %s in LiteralMarkerReplacer.";
+
+ List<Literal> literalList = new LinkedList<Literal>();
+ int literalGlobalIndex = -1;
+
+ public List<Literal> getLiteralList() {
+ return literalList;
+ }
+
+ @Override
+ protected Void visitNode(Node node, Void context) {
+ throw new UnsupportedOperationException(
+ "visitNode of Node {" + node.getClass() + "} is not supported in LiteralMarkerReplacer");
+ }
+
+ @Override
+ protected Void visitQuery(Query node, Void context) {
+ node.getQueryBody().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitQuerySpecification(QuerySpecification node, Void context) {
+ node.getSelect().accept(this, context);
+ node.getWhere().ifPresent(where -> where.accept(this, context));
+ node.getGroupBy().ifPresent(groupBy -> groupBy.accept(this, context));
+ node.getHaving().ifPresent(having -> having.accept(this, context));
+ node.getOrderBy().ifPresent(orderBy -> orderBy.accept(this, context));
+ node.getLimit().ifPresent(limit -> limit.accept(this, context));
+ node.getOffset().ifPresent(offset -> offset.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitSelect(Select node, Void context) {
+ node.getSelectItems().forEach(item -> item.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitAllColumns(AllColumns node, Void context) {
+ node.getTarget().ifPresent(target -> target.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitSingleColumn(SingleColumn node, Void context) {
+ node.getExpression().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitOrderBy(OrderBy node, Void context) {
+ node.getSortItems().forEach(item -> item.getSortKey().accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitGroupBy(GroupBy node, Void context) {
+ for (GroupingElement element : node.getGroupingElements()) {
+ element.getExpressions().forEach(expression -> expression.accept(this, context));
+ }
+ return context;
+ }
+
+ @Override
+ protected Void visitLimit(Limit node, Void context) {
+ node.getRowCount().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitOffset(Offset node, Void context) {
+ node.getRowCount().accept(this, context);
+ return context;
+ }
+
+ // ============================ Expression ============================
+
+ @Override
+ protected Void visitExpression(Expression node, Void context) {
+ // do nothing
+ return context;
+ }
+
+ @Override
+ protected Void visitLogicalExpression(LogicalExpression node, Void context) {
+ node.getTerms().forEach(term -> term.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitComparisonExpression(ComparisonExpression node, Void context) {
+ node.getLeft().accept(this, context);
+ node.getRight().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitIsNullPredicate(IsNullPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitIsNotNullPredicate(IsNotNullPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitFunctionCall(FunctionCall node, Void context) {
+ node.getArguments().forEach(arg -> arg.accept(this, context));
+ return context;
+ }
+
+ @Override
+ protected Void visitLikePredicate(LikePredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitBetweenPredicate(BetweenPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ node.getMin().accept(this, context);
+ node.getMax().accept(this, context);
+ return context;
+ }
+
+ // TODO it can be optimized by an array of expressions
+ @Override
+ protected Void visitInPredicate(InPredicate node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitNotExpression(NotExpression node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitArithmeticBinary(ArithmeticBinaryExpression node, Void context) {
+ node.getLeft().accept(this, context);
+ node.getRight().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitArithmeticUnary(ArithmeticUnaryExpression node, Void context) {
+ node.getValue().accept(this, context);
+ return context;
+ }
+
+ @Override
+ protected Void visitLiteral(Literal node, Void context) {
+ literalList.add(node);
+ node.setLiteralIndex(++literalGlobalIndex);
+ return context;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
index d6222564ec7..e16b3e14c93 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java
@@ -101,6 +101,8 @@ public final class ExpressionFormatter {
() ->
new DecimalFormat("0.###################E0###", new DecimalFormatSymbols(Locale.US)));
+ private static final String LITERAL_MARKER_FORMAT = "LiteralMarker(#%s)";
+
private ExpressionFormatter() {}
public static String formatExpression(Expression expression) {
@@ -178,6 +180,9 @@ public final class ExpressionFormatter {
@Override
protected String visitBooleanLiteral(BooleanLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> String.valueOf(node.getValue()));
@@ -185,6 +190,9 @@ public final class ExpressionFormatter {
@Override
protected String visitStringLiteral(StringLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> formatStringLiteral(node.getValue()));
@@ -192,6 +200,9 @@ public final class ExpressionFormatter {
@Override
protected String visitBinaryLiteral(BinaryLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> "X'" + node.toHexString() + "'");
@@ -209,11 +220,17 @@ public final class ExpressionFormatter {
@Override
protected String visitLongLiteral(LongLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter.map(formatter -> formatter.apply(node)).orElseGet(node::getValue);
}
@Override
protected String visitDoubleLiteral(DoubleLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> doubleFormatter.get().format(node.getValue()));
@@ -221,6 +238,9 @@ public final class ExpressionFormatter {
@Override
protected String visitDecimalLiteral(DecimalLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
// TODO return node value without "DECIMAL '..'" when
@@ -230,6 +250,9 @@ public final class ExpressionFormatter {
@Override
protected String visitGenericLiteral(GenericLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter
.map(formatter -> formatter.apply(node))
.orElseGet(() -> node.getType() + " " + formatStringLiteral(node.getValue()));
@@ -237,6 +260,9 @@ public final class ExpressionFormatter {
@Override
protected String visitNullLiteral(NullLiteral node, Void context) {
+ if (node.isLiteralMarker()) {
+ return String.format(LITERAL_MARKER_FORMAT, node.getLiteralIndex());
+ }
return literalFormatter.map(formatter -> formatter.apply(node)).orElse("null");
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java
new file mode 100644
index 00000000000..4e160814b2e
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.planner;
+
+import org.apache.iotdb.db.protocol.session.IClientSession;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LiteralMarkerReplacer;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.util.SqlFormatter;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.time.ZoneId;
+
+public class PlanCacheTest {
+
+ @Test
+ public void statementGeneralizationTest() {
+
+ String databaseName = "tetsdb";
+ IClientSession clientSession = Mockito.mock(IClientSession.class);
+ Mockito.when(clientSession.getDatabaseName()).thenReturn(databaseName);
+
+ SqlParser sqlParser = new SqlParser();
+ String sql = "select id + 1 from table1 where id > 10 and time=13289078 order by rank+1";
+ long startTime = System.nanoTime();
+ Statement originalStatement =
+ sqlParser.createStatement(sql, ZoneId.systemDefault(), clientSession);
+ System.out.println("Time to parse: " + (System.nanoTime() - startTime));
+
+ LiteralMarkerReplacer literalMarkerReplacer = new LiteralMarkerReplacer();
+ startTime = System.nanoTime();
+ literalMarkerReplacer.process(originalStatement);
+
+ String newSql = SqlFormatter.formatSql(originalStatement);
+ System.out.println("Time to replace: " + (System.nanoTime() - startTime));
+ System.out.println(newSql);
+ }
+}