You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/02/08 01:09:08 UTC
[iotdb] branch lmh/fixFilter1.0 created (now eb3459cbff)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/fixFilter1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at eb3459cbff spotless apply
This branch includes the following new commits:
new ae0e5472d0 [IOTDB-5478] Fix incorrect query filter after binding schema and removing wildcard (#8996)
new eb3459cbff spotless apply
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: spotless apply
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fixFilter1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit eb3459cbff716474c5f39c45f9c733abdac7d6a4
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Feb 8 09:08:45 2023 +0800
spotless apply
---
.../org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
index d3f34f6bc0..d381e89ec8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
@@ -372,7 +372,7 @@ public class QueryStatement extends Statement {
}
}
- if (getHavingCondition() != null) {
+ if (getHavingCondition() != null) {
Expression havingExpression = getHavingCondition().getPredicate();
if (ExpressionAnalyzer.identifyOutputColumnType(havingExpression, true)
!= ResultColumn.ColumnType.AGGREGATION) {
[iotdb] 01/02: [IOTDB-5478] Fix incorrect query filter after binding schema and removing wildcard (#8996)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fixFilter1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae0e5472d070861dba17ea94dd5b38851f771e38
Author: liuminghui233 <36...@users.noreply.github.com>
AuthorDate: Tue Feb 7 14:35:08 2023 +0800
[IOTDB-5478] Fix incorrect query filter after binding schema and removing wildcard (#8996)
(cherry picked from commit 4d8507afa54a2b8f65a488e73e6c9d9ea7b52e78)
---
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 2 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 33 ++-----
.../db/mpp/plan/expression/ExpressionFactory.java | 103 +++++++++++++++++++++
.../db/mpp/plan/statement/crud/QueryStatement.java | 10 +-
.../mpp/plan/analyze/ExpressionAnalyzerTest.java | 83 +++++++++++++++++
5 files changed, 205 insertions(+), 26 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index 42c7aedcc7..99027e495a 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -569,7 +569,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
queryStatement.getHavingCondition().getPredicate(),
queryStatement.getFromComponent().getPrefixPaths(),
schemaTree,
- false);
+ true);
Expression havingExpression =
ExpressionUtils.constructQueryFilter(
conJunctions.stream().distinct().collect(Collectors.toList()));
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index d6766fcbfe..bbf4c1f4b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -485,42 +485,30 @@ public class ExpressionAnalyzer {
* @return the expression list with full path and after binding schema
*/
public static List<Expression> removeWildcardInFilter(
- Expression predicate,
- List<PartialPath> prefixPaths,
- ISchemaTree schemaTree,
- boolean isWhere) {
+ Expression predicate, List<PartialPath> prefixPaths, ISchemaTree schemaTree, boolean isRoot) {
if (predicate instanceof TernaryExpression) {
List<Expression> firstExpressions =
removeWildcardInFilter(
- ((TernaryExpression) predicate).getFirstExpression(),
- prefixPaths,
- schemaTree,
- isWhere);
+ ((TernaryExpression) predicate).getFirstExpression(), prefixPaths, schemaTree, false);
List<Expression> secondExpressions =
removeWildcardInFilter(
((TernaryExpression) predicate).getSecondExpression(),
prefixPaths,
schemaTree,
- isWhere);
+ false);
List<Expression> thirdExpressions =
removeWildcardInFilter(
- ((TernaryExpression) predicate).getThirdExpression(),
- prefixPaths,
- schemaTree,
- isWhere);
+ ((TernaryExpression) predicate).getThirdExpression(), prefixPaths, schemaTree, false);
return reconstructTernaryExpressions(
predicate, firstExpressions, secondExpressions, thirdExpressions);
} else if (predicate instanceof BinaryExpression) {
List<Expression> leftExpressions =
removeWildcardInFilter(
- ((BinaryExpression) predicate).getLeftExpression(), prefixPaths, schemaTree, isWhere);
+ ((BinaryExpression) predicate).getLeftExpression(), prefixPaths, schemaTree, false);
List<Expression> rightExpressions =
removeWildcardInFilter(
- ((BinaryExpression) predicate).getRightExpression(),
- prefixPaths,
- schemaTree,
- isWhere);
- if (predicate.getExpressionType() == ExpressionType.LOGIC_AND) {
+ ((BinaryExpression) predicate).getRightExpression(), prefixPaths, schemaTree, false);
+ if (isRoot && predicate.getExpressionType() == ExpressionType.LOGIC_AND) {
List<Expression> resultExpressions = new ArrayList<>(leftExpressions);
resultExpressions.addAll(rightExpressions);
return resultExpressions;
@@ -530,16 +518,13 @@ public class ExpressionAnalyzer {
} else if (predicate instanceof UnaryExpression) {
List<Expression> childExpressions =
removeWildcardInFilter(
- ((UnaryExpression) predicate).getExpression(), prefixPaths, schemaTree, isWhere);
+ ((UnaryExpression) predicate).getExpression(), prefixPaths, schemaTree, false);
return reconstructUnaryExpressions((UnaryExpression) predicate, childExpressions);
} else if (predicate instanceof FunctionExpression) {
- if (predicate.isBuiltInAggregationFunctionExpression() && isWhere) {
- throw new SemanticException("aggregate functions are not supported in WHERE clause");
- }
List<List<Expression>> extendedExpressions = new ArrayList<>();
for (Expression suffixExpression : predicate.getExpressions()) {
extendedExpressions.add(
- removeWildcardInFilter(suffixExpression, prefixPaths, schemaTree, isWhere));
+ removeWildcardInFilter(suffixExpression, prefixPaths, schemaTree, false));
}
List<List<Expression>> childExpressionsList = new ArrayList<>();
cartesianProduct(extendedExpressions, childExpressionsList, 0, new ArrayList<>());
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ExpressionFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ExpressionFactory.java
new file mode 100644
index 0000000000..170e21db1a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ExpressionFactory.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.expression;
+
+import org.apache.iotdb.common.rpc.thrift.TAggregationType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.mpp.plan.expression.binary.AdditionExpression;
+import org.apache.iotdb.db.mpp.plan.expression.binary.GreaterThanExpression;
+import org.apache.iotdb.db.mpp.plan.expression.binary.LogicAndExpression;
+import org.apache.iotdb.db.mpp.plan.expression.binary.LogicOrExpression;
+import org.apache.iotdb.db.mpp.plan.expression.leaf.ConstantOperand;
+import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
+import org.apache.iotdb.db.mpp.plan.expression.leaf.TimestampOperand;
+import org.apache.iotdb.db.mpp.plan.expression.multi.FunctionExpression;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+
+public class ExpressionFactory {
+
+ public static PartialPath path(String pathStr) throws IllegalPathException {
+ return new PartialPath(pathStr);
+ }
+
+ public static TimeSeriesOperand timeSeries(String pathStr) throws IllegalPathException {
+ PartialPath path = new PartialPath(pathStr);
+ return new TimeSeriesOperand(path);
+ }
+
+ public static ConstantOperand constant(TSDataType dataType, String valueString) {
+ return new ConstantOperand(dataType, valueString);
+ }
+
+ public static ConstantOperand intValue(String valueString) {
+ return new ConstantOperand(TSDataType.INT32, valueString);
+ }
+
+ public static TimestampOperand time() {
+ return new TimestampOperand();
+ }
+
+ public static FunctionExpression function(
+ String functionName,
+ LinkedHashMap<String, String> functionAttributes,
+ Expression... inputExpression) {
+ return new FunctionExpression(functionName, functionAttributes, Arrays.asList(inputExpression));
+ }
+
+ public static FunctionExpression function(String functionName, Expression... inputExpression) {
+ return new FunctionExpression(
+ functionName, new LinkedHashMap<>(), Arrays.asList(inputExpression));
+ }
+
+ public static FunctionExpression count(Expression inputExpression) {
+ return new FunctionExpression(
+ TAggregationType.COUNT.toString(),
+ new LinkedHashMap<>(),
+ Collections.singletonList(inputExpression));
+ }
+
+ public static FunctionExpression sum(Expression inputExpression) {
+ return new FunctionExpression(
+ TAggregationType.SUM.toString(),
+ new LinkedHashMap<>(),
+ Collections.singletonList(inputExpression));
+ }
+
+ public static LogicAndExpression and(Expression leftExpression, Expression rightExpression) {
+ return new LogicAndExpression(leftExpression, rightExpression);
+ }
+
+ public static LogicOrExpression or(Expression leftExpression, Expression rightExpression) {
+ return new LogicOrExpression(leftExpression, rightExpression);
+ }
+
+ public static AdditionExpression add(Expression leftExpression, Expression rightExpression) {
+ return new AdditionExpression(leftExpression, rightExpression);
+ }
+
+ public static GreaterThanExpression gt(Expression leftExpression, Expression rightExpression) {
+ return new GreaterThanExpression(leftExpression, rightExpression);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
index 049c955829..d3f34f6bc0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/QueryStatement.java
@@ -364,7 +364,15 @@ public class QueryStatement extends Statement {
}
}
- if (getHavingCondition() != null) {
+ if (hasWhere()) {
+ Expression whereExpression = getWhereCondition().getPredicate();
+ if (ExpressionAnalyzer.identifyOutputColumnType(whereExpression, true)
+ == ResultColumn.ColumnType.AGGREGATION) {
+ throw new SemanticException("aggregate functions are not supported in WHERE clause");
+ }
+ }
+
+ if (getHavingCondition() != null) {
Expression havingExpression = getHavingCondition().getPredicate();
if (ExpressionAnalyzer.identifyOutputColumnType(havingExpression, true)
!= ResultColumn.ColumnType.AGGREGATION) {
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzerTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzerTest.java
new file mode 100644
index 0000000000..464a5d6378
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzerTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.analyze;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.and;
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.count;
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.gt;
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.intValue;
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.path;
+import static org.apache.iotdb.db.mpp.plan.expression.ExpressionFactory.timeSeries;
+import static org.junit.Assert.assertEquals;
+
+public class ExpressionAnalyzerTest {
+
+ @Test
+ public void testRemoveWildcardInFilter() throws IllegalPathException {
+ ISchemaTree fakeSchemaTree = new FakeSchemaFetcherImpl().fetchSchema(new PathPatternTree());
+ List<PartialPath> prefixPaths = Arrays.asList(path("root.sg.d1"), path("root.sg.d2"));
+
+ assertEquals(
+ Arrays.asList(
+ gt(timeSeries("root.sg.d1.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d2.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d1.s2"), intValue("1")),
+ gt(timeSeries("root.sg.d2.s2"), intValue("1"))),
+ ExpressionAnalyzer.removeWildcardInFilter(
+ and(gt(timeSeries("s1"), intValue("1")), gt(timeSeries("s2"), intValue("1"))),
+ prefixPaths,
+ fakeSchemaTree,
+ true));
+
+ assertEquals(
+ Arrays.asList(
+ count(
+ and(
+ gt(timeSeries("root.sg.d1.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d1.s2"), intValue("1")))),
+ count(
+ and(
+ gt(timeSeries("root.sg.d1.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d2.s2"), intValue("1")))),
+ count(
+ and(
+ gt(timeSeries("root.sg.d2.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d1.s2"), intValue("1")))),
+ count(
+ and(
+ gt(timeSeries("root.sg.d2.s1"), intValue("1")),
+ gt(timeSeries("root.sg.d2.s2"), intValue("1"))))),
+ ExpressionAnalyzer.removeWildcardInFilter(
+ count(and(gt(timeSeries("s1"), intValue("1")), gt(timeSeries("s2"), intValue("1")))),
+ prefixPaths,
+ fakeSchemaTree,
+ true));
+ }
+}