You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/11/22 15:09:43 UTC
(iotdb) branch lmh/refactorFilter updated (f5912d233f3 -> 971404a4fca)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/refactorFilter
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from f5912d233f3 finish PredicateUtils
new 9ab70a5645d rename & add UT test
new 971404a4fca fix sonar
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../queryengine/plan/analyze/PredicateUtils.java | 18 +--
.../ConvertPredicateToTimeFilterVisitor.java | 54 ++++----
.../ConvertPredicateToTimeFilterTest.java | 148 +++++++++++++++++++++
3 files changed, 180 insertions(+), 40 deletions(-)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java
(iotdb) 01/02: rename & add UT test
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/refactorFilter
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9ab70a5645d62d0a86e732c42a884cb5de34cdf0
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Nov 22 23:04:18 2023 +0800
rename & add UT test
---
.../queryengine/plan/analyze/PredicateUtils.java | 18 +--
.../ConvertPredicateToTimeFilterVisitor.java | 2 +-
.../ConvertPredicateToTimeFilterTest.java | 148 +++++++++++++++++++++
3 files changed, 158 insertions(+), 10 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
index 8674d81a908..d2ef00c2c71 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.ternary.TernaryExpression
import org.apache.iotdb.db.queryengine.plan.expression.unary.InExpression;
import org.apache.iotdb.db.queryengine.plan.expression.unary.LogicNotExpression;
import org.apache.iotdb.db.queryengine.plan.expression.unary.UnaryExpression;
+import org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate.ConvertPredicateToTimeFilterVisitor;
import org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate.ReversePredicateVisitor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -174,14 +175,13 @@ public class PredicateUtils {
&& ((ConstantOperand) valueExpression).getDataType() == TSDataType.INT64;
}
- private static boolean checkBetweenConstantSatisfy(
- Expression firstExpression, Expression secondExpression) {
- return firstExpression.isConstantOperand()
- && secondExpression.isConstantOperand()
- && ((ConstantOperand) firstExpression).getDataType() == TSDataType.INT64
- && ((ConstantOperand) secondExpression).getDataType() == TSDataType.INT64
- && (Long.parseLong(((ConstantOperand) firstExpression).getValueString())
- <= Long.parseLong(((ConstantOperand) secondExpression).getValueString()));
+ private static boolean checkBetweenConstantSatisfy(Expression e1, Expression e2) {
+ return e1.isConstantOperand()
+ && e2.isConstantOperand()
+ && ((ConstantOperand) e1).getDataType() == TSDataType.INT64
+ && ((ConstantOperand) e2).getDataType() == TSDataType.INT64
+ && (Long.parseLong(((ConstantOperand) e1).getValueString())
+ <= Long.parseLong(((ConstantOperand) e2).getValueString()));
}
/**
@@ -319,7 +319,7 @@ public class PredicateUtils {
if (predicate == null) {
return null;
}
- return predicate.accept(new ConvertExpressionToTimeFilterVisitor(), null);
+ return predicate.accept(new ConvertPredicateToTimeFilterVisitor(), null);
}
/**
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
index 5daf21f2951..0fc00078203 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
@@ -69,7 +69,7 @@ public class ConvertPredicateToTimeFilterVisitor extends PredicateVisitor<Filter
for (String value : values) {
longValues.add(Long.parseLong(value));
}
- return inExpression.isNotIn() ? TimeFilter.in(longValues) : TimeFilter.notIn(longValues);
+ return inExpression.isNotIn() ? TimeFilter.notIn(longValues) : TimeFilter.in(longValues);
}
private Expression rewriteInExpressionToEqual(InExpression inExpression) {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java
new file mode 100644
index 00000000000..95501bd02fd
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/expression/predicate/ConvertPredicateToTimeFilterTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.expression.predicate;
+
+import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
+import org.apache.iotdb.db.queryengine.plan.expression.Expression;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByTimeParameter;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
+import org.apache.iotdb.tsfile.read.filter.factory.TimeFilter;
+import org.apache.iotdb.tsfile.utils.TimeDuration;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.TimeZone;
+
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.between;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.eq;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.groupByTime;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gt;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.gte;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.in;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.longValue;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.lt;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.lte;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.neq;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.not;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.notBetween;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.notIn;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.or;
+import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time;
+import static org.apache.iotdb.tsfile.read.filter.operator.Not.CONTAIN_NOT_ERR_MSG;
+
+public class ConvertPredicateToTimeFilterTest {
+
+ @Test
+ public void testNormal() {
+ // test comparison
+ testConvertToTimeFilter(eq(time(), longValue(1)), TimeFilter.eq(1));
+ testConvertToTimeFilter(neq(time(), longValue(1)), TimeFilter.notEq(1));
+ testConvertToTimeFilter(gt(time(), longValue(1)), TimeFilter.gt(1));
+ testConvertToTimeFilter(gte(time(), longValue(1)), TimeFilter.gtEq(1));
+ testConvertToTimeFilter(lt(time(), longValue(1)), TimeFilter.lt(1));
+ testConvertToTimeFilter(lte(time(), longValue(1)), TimeFilter.ltEq(1));
+
+ // test between
+ testConvertToTimeFilter(
+ between(time(), longValue(1), longValue(10)), TimeFilter.between(1, 10));
+ testConvertToTimeFilter(
+ notBetween(time(), longValue(1), longValue(10)), TimeFilter.notBetween(1, 10));
+ testConvertToTimeFilter(between(longValue(1), time(), longValue(10)), TimeFilter.ltEq(1));
+ testConvertToTimeFilter(notBetween(longValue(1), time(), longValue(10)), TimeFilter.gt(1));
+ testConvertToTimeFilter(between(longValue(10), longValue(1), time()), TimeFilter.gtEq(10));
+ testConvertToTimeFilter(notBetween(longValue(10), longValue(1), time()), TimeFilter.lt(10));
+
+ // test in
+ LinkedHashSet<String> stringValueSet = new LinkedHashSet<>(Arrays.asList("1", "2", "3"));
+ LinkedHashSet<Long> valueSet = new LinkedHashSet<>(Arrays.asList(1L, 2L, 3L));
+ testConvertToTimeFilter(in(time(), stringValueSet), TimeFilter.in(valueSet));
+ testConvertToTimeFilter(notIn(time(), stringValueSet), TimeFilter.notIn(valueSet));
+
+ // test and/or
+ testConvertToTimeFilter(
+ and(eq(time(), longValue(1L)), eq(time(), longValue(2L))),
+ FilterFactory.and(TimeFilter.eq(1), TimeFilter.eq(2)));
+ testConvertToTimeFilter(
+ or(eq(time(), longValue(1L)), eq(time(), longValue(2L))),
+ FilterFactory.or(TimeFilter.eq(1), TimeFilter.eq(2)));
+
+ // test group by time
+ testConvertToTimeFilter(groupByTime(1, 100, 10, 20), TimeFilter.groupBy(1, 100, 10, 20));
+
+ GroupByTimeParameter parameter =
+ new GroupByTimeParameter(
+ 1, 1000000000, new TimeDuration(1, 0), new TimeDuration(2, 0), true);
+ testConvertToTimeFilter(
+ groupByTime(parameter),
+ TimeFilter.groupByMonth(
+ parameter.getStartTime(),
+ parameter.getEndTime(),
+ parameter.getInterval(),
+ parameter.getSlidingStep(),
+ TimeZone.getTimeZone("+00:00"),
+ TimestampPrecisionUtils.currPrecision));
+ }
+
+ @Test
+ public void testRewrite() {
+ LinkedHashSet<String> stringValueSet = new LinkedHashSet<>(Collections.singletonList("1"));
+ testConvertToTimeFilter(in(time(), stringValueSet), TimeFilter.eq(1));
+ testConvertToTimeFilter(notIn(time(), stringValueSet), TimeFilter.notEq(1));
+
+ testConvertToTimeFilter(between(time(), longValue(1), longValue(1)), TimeFilter.eq(1));
+ testConvertToTimeFilter(notBetween(time(), longValue(1), longValue(1)), TimeFilter.notEq(1));
+
+ testConvertToTimeFilter(groupByTime(1, 100, 10, 10), TimeFilter.between(1, 100));
+ testConvertToTimeFilter(groupByTime(1, 100, 10, 5), TimeFilter.between(1, 100));
+
+ GroupByTimeParameter parameter1 =
+ new GroupByTimeParameter(
+ 1, 1000000000, new TimeDuration(1, 0), new TimeDuration(1, 0), true);
+ testConvertToTimeFilter(
+ groupByTime(parameter1),
+ TimeFilter.between(parameter1.getStartTime(), parameter1.getEndTime()));
+ GroupByTimeParameter parameter2 =
+ new GroupByTimeParameter(
+ 1, 1000000000, new TimeDuration(2, 0), new TimeDuration(1, 0), true);
+ testConvertToTimeFilter(
+ groupByTime(parameter2),
+ TimeFilter.between(parameter2.getStartTime(), parameter2.getEndTime()));
+ }
+
+ @Test
+ public void testNot() {
+ try {
+ PredicateUtils.convertPredicateToTimeFilter(not(eq(time(), longValue(1L))));
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(CONTAIN_NOT_ERR_MSG, e.getMessage());
+ }
+ }
+
+ private void testConvertToTimeFilter(Expression predicate, Filter expectedFilter) {
+ Assert.assertEquals(expectedFilter, PredicateUtils.convertPredicateToTimeFilter(predicate));
+ }
+}
(iotdb) 02/02: fix sonar
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/refactorFilter
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 971404a4fcab6a94c243d2b4bcdae4249d77eb4f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Nov 22 23:09:28 2023 +0800
fix sonar
---
.../ConvertPredicateToTimeFilterVisitor.java | 52 +++++++++-------------
1 file changed, 22 insertions(+), 30 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
index 0fc00078203..336f2409a8f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/visitor/predicate/ConvertPredicateToTimeFilterVisitor.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.expression.visitor.predicate;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType;
+import org.apache.iotdb.db.queryengine.plan.expression.binary.CompareBinaryExpression;
import org.apache.iotdb.db.queryengine.plan.expression.binary.EqualToExpression;
import org.apache.iotdb.db.queryengine.plan.expression.binary.GreaterEqualExpression;
import org.apache.iotdb.db.queryengine.plan.expression.binary.GreaterThanExpression;
@@ -121,62 +122,53 @@ public class ConvertPredicateToTimeFilterVisitor extends PredicateVisitor<Filter
@Override
public Filter visitEqualToExpression(EqualToExpression equalToExpression, Void context) {
- return constructCompareFilter(
- equalToExpression.getExpressionType(),
- equalToExpression.getLeftExpression(),
- equalToExpression.getRightExpression());
+ return visitCompareBinaryExpression(equalToExpression, context);
}
@Override
public Filter visitNonEqualExpression(NonEqualExpression nonEqualExpression, Void context) {
- return constructCompareFilter(
- nonEqualExpression.getExpressionType(),
- nonEqualExpression.getLeftExpression(),
- nonEqualExpression.getRightExpression());
+ return visitCompareBinaryExpression(nonEqualExpression, context);
}
@Override
public Filter visitGreaterThanExpression(
GreaterThanExpression greaterThanExpression, Void context) {
- return constructCompareFilter(
- greaterThanExpression.getExpressionType(),
- greaterThanExpression.getLeftExpression(),
- greaterThanExpression.getRightExpression());
+ return visitCompareBinaryExpression(greaterThanExpression, context);
}
@Override
public Filter visitGreaterEqualExpression(
GreaterEqualExpression greaterEqualExpression, Void context) {
- return constructCompareFilter(
- greaterEqualExpression.getExpressionType(),
- greaterEqualExpression.getLeftExpression(),
- greaterEqualExpression.getRightExpression());
+ return visitCompareBinaryExpression(greaterEqualExpression, context);
}
@Override
public Filter visitLessThanExpression(LessThanExpression lessThanExpression, Void context) {
- return constructCompareFilter(
- lessThanExpression.getExpressionType(),
- lessThanExpression.getLeftExpression(),
- lessThanExpression.getRightExpression());
+ return visitCompareBinaryExpression(lessThanExpression, context);
}
@Override
public Filter visitLessEqualExpression(LessEqualExpression lessEqualExpression, Void context) {
- return constructCompareFilter(
- lessEqualExpression.getExpressionType(),
- lessEqualExpression.getLeftExpression(),
- lessEqualExpression.getRightExpression());
+ return visitCompareBinaryExpression(lessEqualExpression, context);
}
- private Filter constructCompareFilter(
- ExpressionType expressionType, Expression leftExpression, Expression rightExpression) {
- if (leftExpression.getExpressionType().equals(ExpressionType.CONSTANT)) {
- return constructCompareFilter(expressionType, rightExpression, leftExpression);
+ @Override
+ public Filter visitCompareBinaryExpression(
+ CompareBinaryExpression comparisonExpression, Void context) {
+ Expression leftExpression = comparisonExpression.getLeftExpression();
+ Expression rightExpression = comparisonExpression.getRightExpression();
+ ExpressionType expressionType = comparisonExpression.getExpressionType();
+
+ if (leftExpression.getExpressionType().equals(ExpressionType.TIMESTAMP)) {
+ return constructCompareFilter(expressionType, rightExpression);
+ } else {
+ return constructCompareFilter(expressionType, leftExpression);
}
+ }
- long value = Long.parseLong(((ConstantOperand) rightExpression).getValueString());
-
+ private Filter constructCompareFilter(
+ ExpressionType expressionType, Expression constantExpression) {
+ long value = Long.parseLong(((ConstantOperand) constantExpression).getValueString());
switch (expressionType) {
case EQUAL_TO:
return TimeFilter.eq(value);