You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/04/23 09:16:26 UTC
[iotdb] branch rel/1.1 updated: [IOTDB-5784] Incorrect result when querying with offset push-down and time filter
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new 177578964c [IOTDB-5784] Incorrect result when querying with offset push-down and time filter
177578964c is described below
commit 177578964c3f52b4a0cf53d756cd0e7e55e66770
Author: liuminghui233 <36...@users.noreply.github.com>
AuthorDate: Sun Apr 23 17:16:19 2023 +0800
[IOTDB-5784] Incorrect result when querying with offset push-down and time filter
---
.../iotdb/db/it/query/IoTDBPaginationIT.java | 59 ++++++-
.../execution/operator/source/SeriesScanUtil.java | 24 ++-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 4 +
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 11 +-
.../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 3 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 8 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 12 +-
.../source/AlignedSeriesAggregationScanNode.java | 50 +++---
.../plan/node/source/AlignedSeriesScanNode.java | 4 +
.../node/source/SeriesAggregationScanNode.java | 50 +++---
.../node/source/SeriesAggregationSourceNode.java | 49 +++++-
.../planner/plan/node/source/SeriesScanNode.java | 4 +
.../planner/plan/parameter/SeriesScanOptions.java | 5 +-
.../iotdb/db/query/reader/chunk/MemPageReader.java | 12 +-
.../mpp/plan/analyze/QueryTimePartitionTest.java | 14 +-
.../node/process/AggregationNodeSerdeTest.java | 7 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 8 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 7 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 13 +-
.../read/filter/PredicateRemoveNotRewriter.java | 50 ++++++
.../iotdb/tsfile/read/filter/TimeFilter.java | 158 +++++++++---------
.../iotdb/tsfile/read/filter/ValueFilter.java | 179 ++++++++++++---------
.../tsfile/read/filter/basic/BinaryFilter.java | 5 -
.../iotdb/tsfile/read/filter/basic/Filter.java | 11 +-
.../tsfile/read/filter/factory/FilterFactory.java | 10 +-
.../tsfile/read/filter/operator/AndFilter.java | 11 ++
.../iotdb/tsfile/read/filter/operator/Between.java | 127 +++++++++------
.../iotdb/tsfile/read/filter/operator/Eq.java | 25 ++-
.../iotdb/tsfile/read/filter/operator/Gt.java | 23 ++-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 23 ++-
.../iotdb/tsfile/read/filter/operator/In.java | 34 +++-
.../iotdb/tsfile/read/filter/operator/Like.java | 55 +++++--
.../iotdb/tsfile/read/filter/operator/Lt.java | 23 ++-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 23 ++-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 25 ++-
.../tsfile/read/filter/operator/NotFilter.java | 26 +--
.../tsfile/read/filter/operator/OrFilter.java | 11 ++
.../iotdb/tsfile/read/filter/operator/Regexp.java | 57 +++++--
.../iotdb/tsfile/read/reader/page/PageReader.java | 12 +-
.../tsfile/read/filter/FilterSerializeTest.java | 21 ++-
.../read/filter/MinTimeMaxTimeFilterTest.java | 42 +++--
.../iotdb/tsfile/read/filter/OperatorTest.java | 4 +-
.../filter/PredicateRemoveNotRewriterTest.java | 121 ++++++++++++++
.../tsfile/read/filter/StatisticsFilterTest.java | 113 ++++++++++++-
45 files changed, 1109 insertions(+), 429 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java
index 0e1fb007f5..f9d9a70a8a 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java
@@ -76,11 +76,36 @@ public class IoTDBPaginationIT {
"insert into root.vehicle.d0(timestamp,s2) values(102,10.00)",
"insert into root.vehicle.d0(timestamp,s2) values(105,11.11)",
"insert into root.vehicle.d0(timestamp,s2) values(1000,1000.11)",
- "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T08:00:00+08:00, 100)"
+ "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T08:00:00+08:00, 100)",
+ "CREATE DATABASE root.db",
+ "CREATE TIMESERIES root.db.d1.s1 INT32",
+ "insert into root.db.d1(timestamp,s1) values(0, 0)",
+ "insert into root.db.d1(timestamp,s1) values(1, 1)",
+ "insert into root.db.d1(timestamp,s1) values(2, 2)",
+ "insert into root.db.d1(timestamp,s1) values(3, 3)",
+ "insert into root.db.d1(timestamp,s1) values(4, 4)",
+ "flush",
+ "insert into root.db.d1(timestamp,s1) values(5, 5)",
+ "insert into root.db.d1(timestamp,s1) values(6, 6)",
+ "insert into root.db.d1(timestamp,s1) values(7, 7)",
+ "insert into root.db.d1(timestamp,s1) values(8, 8)",
+ "insert into root.db.d1(timestamp,s1) values(9, 9)",
+ "flush",
+ "insert into root.db.d1(timestamp,s1) values(10, 10)",
+ "insert into root.db.d1(timestamp,s1) values(11, 11)",
+ "insert into root.db.d1(timestamp,s1) values(12, 12)",
+ "insert into root.db.d1(timestamp,s1) values(13, 13)",
+ "insert into root.db.d1(timestamp,s1) values(14, 14)"
};
@BeforeClass
public static void setUp() throws InterruptedException {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setEnableSeqSpaceCompaction(false)
+ .setEnableUnseqSpaceCompaction(false)
+ .setEnableCrossSpaceCompaction(false);
EnvFactory.getEnv().initClusterEnvironment();
prepareData(SQLs);
}
@@ -135,4 +160,36 @@ public class IoTDBPaginationIT {
resultSetEqualTest(querySQLs.get(0), expectHeaders.get(0), retArrays.get(0));
}
}
+
+ @Test
+ public void limitOffsetPushDownTest() {
+ String expectedHeader = "Time,root.db.d1.s1,";
+ String[] retArray =
+ new String[] {
+ "3,3,",
+ };
+ resultSetEqualTest(
+ "select s1 from root.db.d1 where time > 1 offset 1 limit 1", expectedHeader, retArray);
+
+ retArray =
+ new String[] {
+ "5,5,",
+ };
+ resultSetEqualTest(
+ "select s1 from root.db.d1 where time > 1 offset 3 limit 1", expectedHeader, retArray);
+
+ retArray =
+ new String[] {
+ "7,7,",
+ };
+ resultSetEqualTest(
+ "select s1 from root.db.d1 where time > 1 offset 5 limit 1", expectedHeader, retArray);
+
+ retArray =
+ new String[] {
+ "12,12,",
+ };
+ resultSetEqualTest(
+ "select s1 from root.db.d1 where time > 1 offset 10 limit 1", expectedHeader, retArray);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
index d42c2ff931..d3988be366 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
@@ -288,17 +288,15 @@ public class SeriesScanUtil {
protected void filterFirstChunkMetadata() throws IOException {
if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) {
Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstChunkMetadata.getStatistics())) {
- skipCurrentChunk();
- }
- // TODO implement allSatisfied interface for filter, then we can still skip offset.
- } else {
- long rowCount = firstChunkMetadata.getStatistics().getCount();
+ Statistics statistics = firstChunkMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
if (paginationController.hasCurOffset(rowCount)) {
skipCurrentChunk();
paginationController.consumeOffset(rowCount);
}
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentChunk();
}
}
}
@@ -1041,17 +1039,15 @@ public class SeriesScanUtil {
&& !isFileOverlapped()
&& !firstTimeSeriesMetadata.isModified()) {
Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) {
- skipCurrentFile();
- }
- // TODO implement allSatisfied interface for filter, then we can still skip offset.
- } else {
- long rowCount = firstTimeSeriesMetadata.getStatistics().getCount();
+ Statistics statistics = firstTimeSeriesMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
if (paginationController.hasCurOffset(rowCount)) {
skipCurrentFile();
paginationController.consumeOffset(rowCount);
}
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentFile();
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index f929573fb1..d2b44cfdd4 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -144,6 +144,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter;
+import org.apache.iotdb.tsfile.read.filter.PredicateRemoveNotRewriter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -373,6 +374,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Pair<Filter, Boolean> resultPair =
ExpressionAnalyzer.extractGlobalTimeFilter(predicate, true, true);
globalTimeFilter = resultPair.left;
+ if (globalTimeFilter != null) {
+ globalTimeFilter = PredicateRemoveNotRewriter.rewrite(globalTimeFilter);
+ }
hasValueFilter = resultPair.right;
predicate = ExpressionAnalyzer.evaluatePredicate(predicate);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
index 915afc8b06..d274ae03ea 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java
@@ -930,12 +930,11 @@ public class ExpressionAnalyzer {
} else if (predicate.getExpressionType().equals(ExpressionType.IN)) {
Expression timeExpression = ((InExpression) predicate).getExpression();
if (timeExpression.getExpressionType().equals(ExpressionType.TIMESTAMP)) {
- return new Pair<>(
- TimeFilter.in(
- ((InExpression) predicate)
- .getValues().stream().map(Long::parseLong).collect(Collectors.toSet()),
- ((InExpression) predicate).isNotIn()),
- false);
+ boolean not = ((InExpression) predicate).isNotIn();
+ Set<Long> values =
+ ((InExpression) predicate)
+ .getValues().stream().map(Long::parseLong).collect(Collectors.toSet());
+ return new Pair<>(not ? TimeFilter.notIn(values) : TimeFilter.in(values), false);
}
return new Pair<>(null, true);
} else if (predicate.getExpressionType().equals(ExpressionType.TIMESERIES)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java
index 62df923970..593c3fb278 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java
@@ -365,7 +365,8 @@ public class ExpressionUtils {
&& ((ConstantOperand) secondExpression).getDataType() == TSDataType.INT64) {
long value1 = Long.parseLong(((ConstantOperand) firstExpression).getValueString());
long value2 = Long.parseLong(((ConstantOperand) secondExpression).getValueString());
- return new Pair<>(TimeFilter.between(value1, value2, not), false);
+ return new Pair<>(
+ not ? TimeFilter.notBetween(value1, value2) : TimeFilter.between(value1, value2), false);
} else {
return new Pair<>(null, true);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index 84d121be41..e36e29aece 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -177,12 +177,16 @@ public class LogicalPlanBuilder {
new SeriesScanNode(
context.getQueryId().genPlanNodeId(), (MeasurementPath) path, scanOrder);
seriesScanNode.setTimeFilter(timeFilter);
+ // TODO: push down value filter
+ seriesScanNode.setValueFilter(timeFilter);
sourceNodeList.add(seriesScanNode);
} else if (path instanceof AlignedPath) { // aligned series
AlignedSeriesScanNode alignedSeriesScanNode =
new AlignedSeriesScanNode(
context.getQueryId().genPlanNodeId(), (AlignedPath) path, scanOrder);
alignedSeriesScanNode.setTimeFilter(timeFilter);
+ // TODO: push down value filter
+ alignedSeriesScanNode.setValueFilter(timeFilter);
sourceNodeList.add(alignedSeriesScanNode);
} else {
throw new IllegalArgumentException("unexpected path type");
@@ -790,6 +794,8 @@ public class LogicalPlanBuilder {
scanOrder,
groupByTimeParameter);
seriesAggregationScanNode.setTimeFilter(timeFilter);
+ // TODO: push down value filter
+ seriesAggregationScanNode.setValueFilter(timeFilter);
return seriesAggregationScanNode;
} else if (selectPath instanceof AlignedPath) { // aligned series
AlignedSeriesAggregationScanNode alignedSeriesAggregationScanNode =
@@ -800,6 +806,8 @@ public class LogicalPlanBuilder {
scanOrder,
groupByTimeParameter);
alignedSeriesAggregationScanNode.setTimeFilter(timeFilter);
+ // TODO: push down value filter
+ alignedSeriesAggregationScanNode.setValueFilter(timeFilter);
return alignedSeriesAggregationScanNode;
} else {
throw new IllegalArgumentException("unexpected path type");
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
index 2742b4bce2..cd5ea260f4 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
@@ -285,7 +285,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
seriesScanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy());
}
if (valueFilter != null) {
- seriesScanOptionsBuilder.withGlobalTimeFilter(valueFilter.copy());
+ seriesScanOptionsBuilder.withQueryFilter(valueFilter.copy());
}
seriesScanOptionsBuilder.withAllSensors(
context.getAllSensors(seriesPath.getDevice(), seriesPath.getMeasurement()));
@@ -327,7 +327,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
seriesScanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy());
}
if (valueFilter != null) {
- seriesScanOptionsBuilder.withGlobalTimeFilter(valueFilter.copy());
+ seriesScanOptionsBuilder.withQueryFilter(valueFilter.copy());
}
seriesScanOptionsBuilder.withLimit(node.getLimit());
seriesScanOptionsBuilder.withOffset(node.getOffset());
@@ -385,12 +385,16 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
node.getAggregationDescriptorList(), timeRangeIterator, context.getTypeProvider());
Filter timeFilter = node.getTimeFilter();
+ Filter valueFilter = node.getValueFilter();
SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder();
scanOptionsBuilder.withAllSensors(
context.getAllSensors(seriesPath.getDevice(), seriesPath.getMeasurement()));
if (timeFilter != null) {
scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy());
}
+ if (valueFilter != null) {
+ scanOptionsBuilder.withQueryFilter(valueFilter.copy());
+ }
SeriesAggregationScanOperator aggregateScanOperator =
new SeriesAggregationScanOperator(
@@ -458,11 +462,15 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
node.getAggregationDescriptorList(), timeRangeIterator, context.getTypeProvider());
Filter timeFilter = node.getTimeFilter();
+ Filter valueFilter = node.getValueFilter();
SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder();
scanOptionsBuilder.withAllSensors(new HashSet<>(seriesPath.getMeasurementList()));
if (timeFilter != null) {
scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy());
}
+ if (valueFilter != null) {
+ scanOptionsBuilder.withQueryFilter(valueFilter.copy());
+ }
AlignedSeriesAggregationScanOperator seriesAggregationScanOperator =
new AlignedSeriesAggregationScanOperator(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java
index 319390f6c0..19dc93a5e8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java
@@ -83,10 +83,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
List<AggregationDescriptor> aggregationDescriptorList,
Ordering scanOrder,
@Nullable Filter timeFilter,
+ @Nullable Filter valueFilter,
@Nullable GroupByTimeParameter groupByTimeParameter,
TRegionReplicaSet dataRegionReplicaSet) {
this(id, alignedPath, aggregationDescriptorList, scanOrder, groupByTimeParameter);
this.timeFilter = timeFilter;
+ this.valueFilter = valueFilter;
this.regionReplicaSet = dataRegionReplicaSet;
}
@@ -99,22 +101,6 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
return scanOrder;
}
- @Override
- @Nullable
- public Filter getTimeFilter() {
- return timeFilter;
- }
-
- public void setTimeFilter(@Nullable Filter timeFilter) {
- this.timeFilter = timeFilter;
- }
-
- @Override
- @Nullable
- public GroupByTimeParameter getGroupByTimeParameter() {
- return groupByTimeParameter;
- }
-
@Override
public void open() throws Exception {}
@@ -155,6 +141,7 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
getAggregationDescriptorList(),
getScanOrder(),
getTimeFilter(),
+ getValueFilter(),
getGroupByTimeParameter(),
getRegionReplicaSet());
}
@@ -187,6 +174,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
ReadWriteIOUtils.write((byte) 1, byteBuffer);
timeFilter.serialize(byteBuffer);
}
+ if (valueFilter == null) {
+ ReadWriteIOUtils.write((byte) 0, byteBuffer);
+ } else {
+ ReadWriteIOUtils.write((byte) 1, byteBuffer);
+ valueFilter.serialize(byteBuffer);
+ }
if (groupByTimeParameter == null) {
ReadWriteIOUtils.write((byte) 0, byteBuffer);
} else {
@@ -210,6 +203,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
ReadWriteIOUtils.write((byte) 1, stream);
timeFilter.serialize(stream);
}
+ if (valueFilter == null) {
+ ReadWriteIOUtils.write((byte) 0, stream);
+ } else {
+ ReadWriteIOUtils.write((byte) 1, stream);
+ valueFilter.serialize(stream);
+ }
if (groupByTimeParameter == null) {
ReadWriteIOUtils.write((byte) 0, stream);
} else {
@@ -232,6 +231,11 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
timeFilter = FilterFactory.deserialize(byteBuffer);
}
isNull = ReadWriteIOUtils.readByte(byteBuffer);
+ Filter valueFilter = null;
+ if (isNull == 1) {
+ valueFilter = FilterFactory.deserialize(byteBuffer);
+ }
+ isNull = ReadWriteIOUtils.readByte(byteBuffer);
GroupByTimeParameter groupByTimeParameter = null;
if (isNull == 1) {
groupByTimeParameter = GroupByTimeParameter.deserialize(byteBuffer);
@@ -243,6 +247,7 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
aggregationDescriptorList,
scanOrder,
timeFilter,
+ valueFilter,
groupByTimeParameter,
null);
}
@@ -260,23 +265,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod
}
AlignedSeriesAggregationScanNode that = (AlignedSeriesAggregationScanNode) o;
return alignedPath.equals(that.alignedPath)
- && aggregationDescriptorList.equals(that.aggregationDescriptorList)
- && scanOrder == that.scanOrder
- && Objects.equals(timeFilter, that.timeFilter)
- && Objects.equals(groupByTimeParameter, that.groupByTimeParameter)
&& Objects.equals(regionReplicaSet, that.regionReplicaSet);
}
@Override
public int hashCode() {
- return Objects.hash(
- super.hashCode(),
- alignedPath,
- aggregationDescriptorList,
- scanOrder,
- timeFilter,
- groupByTimeParameter,
- regionReplicaSet);
+ return Objects.hash(super.hashCode(), alignedPath, regionReplicaSet);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java
index 09387d0e5b..6629a287a1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java
@@ -119,6 +119,10 @@ public class AlignedSeriesScanNode extends SeriesSourceNode {
return valueFilter;
}
+ public void setValueFilter(@Nullable Filter valueFilter) {
+ this.valueFilter = valueFilter;
+ }
+
public long getLimit() {
return limit;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java
index 2ea730b747..c5b6ac8727 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java
@@ -95,10 +95,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
List<AggregationDescriptor> aggregationDescriptorList,
Ordering scanOrder,
@Nullable Filter timeFilter,
+ @Nullable Filter valueFilter,
@Nullable GroupByTimeParameter groupByTimeParameter,
TRegionReplicaSet dataRegionReplicaSet) {
this(id, seriesPath, aggregationDescriptorList, scanOrder, groupByTimeParameter);
this.timeFilter = timeFilter;
+ this.valueFilter = valueFilter;
this.regionReplicaSet = dataRegionReplicaSet;
}
@@ -107,22 +109,6 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
return scanOrder;
}
- @Override
- @Nullable
- public Filter getTimeFilter() {
- return timeFilter;
- }
-
- public void setTimeFilter(@Nullable Filter timeFilter) {
- this.timeFilter = timeFilter;
- }
-
- @Override
- @Nullable
- public GroupByTimeParameter getGroupByTimeParameter() {
- return groupByTimeParameter;
- }
-
public MeasurementPath getSeriesPath() {
return seriesPath;
}
@@ -150,6 +136,7 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
getAggregationDescriptorList(),
getScanOrder(),
getTimeFilter(),
+ getValueFilter(),
getGroupByTimeParameter(),
getRegionReplicaSet());
}
@@ -198,6 +185,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
ReadWriteIOUtils.write((byte) 1, byteBuffer);
timeFilter.serialize(byteBuffer);
}
+ if (valueFilter == null) {
+ ReadWriteIOUtils.write((byte) 0, byteBuffer);
+ } else {
+ ReadWriteIOUtils.write((byte) 1, byteBuffer);
+ valueFilter.serialize(byteBuffer);
+ }
if (groupByTimeParameter == null) {
ReadWriteIOUtils.write((byte) 0, byteBuffer);
} else {
@@ -221,6 +214,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
ReadWriteIOUtils.write((byte) 1, stream);
timeFilter.serialize(stream);
}
+ if (valueFilter == null) {
+ ReadWriteIOUtils.write((byte) 0, stream);
+ } else {
+ ReadWriteIOUtils.write((byte) 1, stream);
+ valueFilter.serialize(stream);
+ }
if (groupByTimeParameter == null) {
ReadWriteIOUtils.write((byte) 0, stream);
} else {
@@ -243,6 +242,11 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
timeFilter = FilterFactory.deserialize(byteBuffer);
}
isNull = ReadWriteIOUtils.readByte(byteBuffer);
+ Filter valueFilter = null;
+ if (isNull == 1) {
+ valueFilter = FilterFactory.deserialize(byteBuffer);
+ }
+ isNull = ReadWriteIOUtils.readByte(byteBuffer);
GroupByTimeParameter groupByTimeParameter = null;
if (isNull == 1) {
groupByTimeParameter = GroupByTimeParameter.deserialize(byteBuffer);
@@ -254,6 +258,7 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
aggregationDescriptorList,
scanOrder,
timeFilter,
+ valueFilter,
groupByTimeParameter,
null);
}
@@ -271,23 +276,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode {
}
SeriesAggregationScanNode that = (SeriesAggregationScanNode) o;
return seriesPath.equals(that.seriesPath)
- && aggregationDescriptorList.equals(that.aggregationDescriptorList)
- && scanOrder == that.scanOrder
- && Objects.equals(timeFilter, that.timeFilter)
- && Objects.equals(groupByTimeParameter, that.groupByTimeParameter)
&& Objects.equals(regionReplicaSet, that.regionReplicaSet);
}
@Override
public int hashCode() {
- return Objects.hash(
- super.hashCode(),
- seriesPath,
- aggregationDescriptorList,
- scanOrder,
- timeFilter,
- groupByTimeParameter,
- regionReplicaSet);
+ return Objects.hash(super.hashCode(), seriesPath, regionReplicaSet);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java
index 183c44cdc9..4857292fde 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import javax.annotation.Nullable;
import java.util.List;
+import java.util.Objects;
public abstract class SeriesAggregationSourceNode extends SeriesSourceNode {
@@ -40,9 +41,12 @@ public abstract class SeriesAggregationSourceNode extends SeriesSourceNode {
// The default order is TIMESTAMP_ASC, which means "order by timestamp asc"
protected Ordering scanOrder = Ordering.ASC;
- // time filter for current series, could be null if doesn't exist
+ // time filter for current series, could be null if it doesn't exist
@Nullable protected Filter timeFilter;
+ // push-downing query filter for current series, could be null if it doesn't exist
+ @Nullable protected Filter valueFilter;
+
// The parameter of `group by time`
// Its value will be null if there is no `group by time` clause,
@Nullable protected GroupByTimeParameter groupByTimeParameter;
@@ -70,8 +74,51 @@ public abstract class SeriesAggregationSourceNode extends SeriesSourceNode {
return timeFilter;
}
+ public void setTimeFilter(@Nullable Filter timeFilter) {
+ this.timeFilter = timeFilter;
+ }
+
+ @Nullable
+ public Filter getValueFilter() {
+ return valueFilter;
+ }
+
+ public void setValueFilter(@Nullable Filter valueFilter) {
+ this.valueFilter = valueFilter;
+ }
+
@Nullable
public GroupByTimeParameter getGroupByTimeParameter() {
return groupByTimeParameter;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ SeriesAggregationSourceNode that = (SeriesAggregationSourceNode) o;
+ return aggregationDescriptorList.equals(that.aggregationDescriptorList)
+ && scanOrder == that.scanOrder
+ && Objects.equals(timeFilter, that.timeFilter)
+ && Objects.equals(valueFilter, that.valueFilter)
+ && Objects.equals(groupByTimeParameter, that.groupByTimeParameter);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ super.hashCode(),
+ aggregationDescriptorList,
+ scanOrder,
+ timeFilter,
+ valueFilter,
+ groupByTimeParameter);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java
index 6458d5af0b..0572544430 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java
@@ -159,6 +159,10 @@ public class SeriesScanNode extends SeriesSourceNode {
return valueFilter;
}
+ public void setValueFilter(@Nullable Filter valueFilter) {
+ this.valueFilter = valueFilter;
+ }
+
@Override
public List<PlanNode> getChildren() {
return ImmutableList.of();
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java
index 7e3bcd6f29..c5032a0786 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.tsfile.read.reader.series.PaginationController;
import java.util.Collections;
import java.util.HashSet;
-import java.util.Objects;
import java.util.Set;
public class SeriesScanOptions {
@@ -50,9 +49,7 @@ public class SeriesScanOptions {
long offset,
Set<String> allSensors) {
this.globalTimeFilter = globalTimeFilter;
- if (!Objects.equals(globalTimeFilter, queryFilter)) {
- this.queryFilter = queryFilter;
- }
+ this.queryFilter = queryFilter;
this.limit = limit;
this.offset = offset;
this.allSensors = allSensors;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
index ad6916103b..4e4c4916ed 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java
@@ -94,16 +94,18 @@ public class MemPageReader implements IPageReader {
}
private boolean pageSatisfy() {
- if (valueFilter != null) {
- return valueFilter.satisfy(getStatistics());
- } else {
- long rowCount = getStatistics().getCount();
+ Statistics statistics = getStatistics();
+ if (valueFilter == null || valueFilter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
if (paginationController.hasCurOffset(rowCount)) {
paginationController.consumeOffset(rowCount);
return false;
+ } else {
+ return true;
}
+ } else {
+ return valueFilter.satisfy(statistics);
}
- return true;
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java
index e795634438..c7951571a0 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java
@@ -186,14 +186,14 @@ public class QueryTimePartitionTest {
public void testBetweenTimeFilter() {
// time between 10 and 20
- TimeFilter.TimeBetween filter = TimeFilter.between(10, 20, false);
+ TimeFilter.TimeBetween filter = TimeFilter.between(10, 20);
List<TimeRange> timeRangeList = filter.getTimeRanges();
assertEquals(1, timeRangeList.size());
assertEquals(10, timeRangeList.get(0).getMin());
assertEquals(20, timeRangeList.get(0).getMax());
// time not between 10 and 20
- filter = TimeFilter.between(10, 20, true);
+ filter = TimeFilter.notBetween(10, 20);
timeRangeList = filter.getTimeRanges();
assertEquals(2, timeRangeList.size());
assertEquals(Long.MIN_VALUE, timeRangeList.get(0).getMin());
@@ -202,14 +202,14 @@ public class QueryTimePartitionTest {
assertEquals(Long.MAX_VALUE, timeRangeList.get(1).getMax());
// time not between 10 and Long.MAX_VALUE
- filter = TimeFilter.between(10, Long.MAX_VALUE, true);
+ filter = TimeFilter.notBetween(10, Long.MAX_VALUE);
timeRangeList = filter.getTimeRanges();
assertEquals(1, timeRangeList.size());
assertEquals(Long.MIN_VALUE, timeRangeList.get(0).getMin());
assertEquals(9, timeRangeList.get(0).getMax());
// time not between Long.MIN_VALUE and 20
- filter = TimeFilter.between(Long.MIN_VALUE, 20, true);
+ filter = TimeFilter.notBetween(Long.MIN_VALUE, 20);
timeRangeList = filter.getTimeRanges();
assertEquals(1, timeRangeList.size());
assertEquals(21, timeRangeList.get(0).getMin());
@@ -441,8 +441,7 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilter.between(
IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1,
- IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(),
- false));
+ IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval()));
expected =
Arrays.asList(
new TTimePartitionSlot(0),
@@ -481,8 +480,7 @@ public class QueryTimePartitionTest {
getTimePartitionSlotList(
TimeFilter.between(
IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(),
- IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1,
- false));
+ IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1));
expected =
Collections.singletonList(
new TTimePartitionSlot(
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java
index 5e78bd06d6..abee7fea83 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java
@@ -32,7 +32,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.mpp.plan.statement.component.Ordering;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.filter.operator.In;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.apache.commons.compress.utils.Sets;
import org.junit.Test;
@@ -42,7 +43,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import static org.apache.iotdb.tsfile.read.filter.factory.FilterType.VALUE_FILTER;
import static org.junit.Assert.assertEquals;
public class AggregationNodeSerdeTest {
@@ -62,7 +62,8 @@ public class AggregationNodeSerdeTest {
Collections.singletonList(
new TimeSeriesOperand(new PartialPath("root.sg.d1.s1"))))),
Ordering.ASC,
- new In<>(Sets.newHashSet("s1", "s2"), VALUE_FILTER, true),
+ TimeFilter.gt(100L),
+ ValueFilter.in(Sets.newHashSet("s1", "s2")),
groupByTimeParameter,
null);
AggregationNode aggregationNode =
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java
index 87af66e497..23346894f3 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java
@@ -33,6 +33,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.CrossSeriesAggregatio
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.mpp.plan.statement.component.Ordering;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.junit.Test;
@@ -62,7 +64,8 @@ public class GroupByLevelNodeSerdeTest {
Collections.singletonList(
new TimeSeriesOperand(new PartialPath("root.sg.d1.s1"))))),
Ordering.ASC,
- null,
+ TimeFilter.gt(100L),
+ ValueFilter.gt(100),
groupByTimeParameter,
null);
SeriesAggregationScanNode seriesAggregationScanNode2 =
@@ -76,7 +79,8 @@ public class GroupByLevelNodeSerdeTest {
Collections.singletonList(
new TimeSeriesOperand(new PartialPath("root.sg.d2.s1"))))),
Ordering.ASC,
- null,
+ TimeFilter.gt(100L),
+ ValueFilter.gt(100),
groupByTimeParameter,
null);
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java
index b0c0c139e5..5c38ff3aaf 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java
@@ -34,6 +34,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.CrossSeriesAggregatio
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.mpp.plan.statement.component.Ordering;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.junit.Assert;
import org.junit.Test;
@@ -102,7 +104,8 @@ public class GroupByTagNodeSerdeTest {
new MeasurementPath("root.sg.d1.s1", TSDataType.INT32),
Arrays.asList(s1MaxTimePartial, s1AvgTimePartial),
Ordering.ASC,
- null,
+ TimeFilter.gt(100L),
+ ValueFilter.gt(100),
groupByTimeParameter,
null)),
groupByTimeParameter,
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java
index d7f1a614b6..a0ac6ff2c3 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java
@@ -32,7 +32,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.mpp.plan.statement.component.Ordering;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.filter.operator.In;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.apache.commons.compress.utils.Sets;
import org.junit.Test;
@@ -42,7 +43,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import static org.apache.iotdb.tsfile.read.filter.factory.FilterType.VALUE_FILTER;
import static org.junit.Assert.assertEquals;
public class SeriesAggregationScanNodeSerdeTest {
@@ -62,7 +62,8 @@ public class SeriesAggregationScanNodeSerdeTest {
new MeasurementPath("root.sg.d1.s1", TSDataType.BOOLEAN),
aggregationDescriptorList,
Ordering.ASC,
- new In<>(Sets.newHashSet("s1", "s2"), VALUE_FILTER, true),
+ TimeFilter.gt(100L),
+ ValueFilter.in(Sets.newHashSet("s1", "s2")),
groupByTimeParameter,
null);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
index 851d122bab..9ef5750de6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
@@ -53,6 +53,11 @@ public class GroupByFilter implements Filter, Serializable {
return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ }
+
@Override
public boolean satisfy(long time, Object value) {
if (time < startTime || time >= endTime) {
@@ -101,7 +106,8 @@ public class GroupByFilter implements Filter, Serializable {
@Override
public String toString() {
- return "GroupByFilter{}";
+ return String.format(
+ "GroupByFilter{[%d, %d], %d, %d}", startTime, endTime, interval, slidingStep);
}
@Override
@@ -161,4 +167,9 @@ public class GroupByFilter implements Filter, Serializable {
? Collections.emptyList()
: Collections.singletonList(new TimeRange(startTime, endTime - 1));
}
+
+ @Override
+ public Filter reverse() {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java
new file mode 100644
index 0000000000..4a66d032f9
--- /dev/null
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read.filter;
+
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId;
+import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
+import org.apache.iotdb.tsfile.read.filter.operator.NotFilter;
+import org.apache.iotdb.tsfile.read.filter.operator.OrFilter;
+
+public class PredicateRemoveNotRewriter {
+
+ public static Filter rewrite(Filter filter) {
+ return removeNot(filter);
+ }
+
+ private static Filter removeNot(Filter filter) {
+ FilterSerializeId filterType = filter.getSerializeId();
+ switch (filterType) {
+ case AND:
+ return FilterFactory.and(
+ removeNot(((AndFilter) filter).getLeft()), removeNot(((AndFilter) filter).getRight()));
+ case OR:
+ return FilterFactory.or(
+ removeNot(((OrFilter) filter).getLeft()), removeNot(((OrFilter) filter).getRight()));
+ case NOT:
+ return ((NotFilter) filter).getFilter().reverse();
+ default:
+ return filter;
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java
index e33b617e60..0cbd94dc0a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.tsfile.read.filter.operator.In;
import org.apache.iotdb.tsfile.read.filter.operator.Lt;
import org.apache.iotdb.tsfile.read.filter.operator.LtEq;
import org.apache.iotdb.tsfile.read.filter.operator.NotEq;
-import org.apache.iotdb.tsfile.read.filter.operator.NotFilter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -42,10 +41,6 @@ public class TimeFilter {
private TimeFilter() {}
- public static TimeEq eq(long value) {
- return new TimeEq(value);
- }
-
public static TimeGt gt(long value) {
return new TimeGt(value);
}
@@ -62,160 +57,163 @@ public class TimeFilter {
return new TimeLtEq(value);
}
- public static TimeNotFilter not(Filter filter) {
- return new TimeNotFilter(filter);
+ public static TimeEq eq(long value) {
+ return new TimeEq(value);
}
public static TimeNotEq notEq(long value) {
return new TimeNotEq(value);
}
- public static TimeIn in(Set<Long> values, boolean not) {
- return new TimeIn(values, not);
+ public static TimeBetween between(long value1, long value2) {
+ return new TimeBetween(value1, value2, false);
}
- public static TimeBetween between(long value1, long value2, boolean not) {
- return new TimeBetween(value1, value2, not);
+ public static TimeBetween notBetween(long value1, long value2) {
+ return new TimeBetween(value1, value2, true);
}
- public static class TimeBetween extends Between {
-
- private TimeBetween(long value1, long value2, boolean not) {
- super(value1, value2, FilterType.TIME_FILTER, not);
- }
+ public static TimeIn in(Set<Long> values) {
+ return new TimeIn(values, false);
+ }
- @Override
- public List<TimeRange> getTimeRanges() {
- long left = (long) value1;
- long right = (long) value2;
- if (not) {
- List<TimeRange> res = new ArrayList<>();
- if (left != Long.MIN_VALUE) {
- res.add(new TimeRange(Long.MIN_VALUE, left - 1));
- }
- if (right != Long.MAX_VALUE) {
- res.add(new TimeRange(right + 1, Long.MAX_VALUE));
- }
- return res;
- } else {
- return Collections.singletonList(new TimeRange(left, right));
- }
- }
+ public static TimeIn notIn(Set<Long> values) {
+ return new TimeIn(values, true);
}
- public static class TimeIn extends In {
+ public static class TimeGt extends Gt<Long> {
- private TimeIn(Set<Long> values, boolean not) {
- super(values, FilterType.TIME_FILTER, not);
+ private TimeGt(long value) {
+ super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- return ((Set<Long>) values)
- .stream()
- .map(
- l -> {
- long time = l;
- return new TimeRange(time, time);
- })
- .collect(Collectors.toList());
+ long left = value;
+ if (left != Long.MAX_VALUE) {
+ return Collections.singletonList(new TimeRange(left + 1, Long.MAX_VALUE));
+ } else {
+ return Collections.emptyList();
+ }
}
}
- public static class TimeEq extends Eq {
+ public static class TimeGtEq extends GtEq<Long> {
- private TimeEq(long value) {
+ private TimeGtEq(long value) {
super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- return Collections.singletonList(new TimeRange((long) value, (long) value));
+ return Collections.singletonList(new TimeRange(value, Long.MAX_VALUE));
}
}
- public static class TimeNotEq extends NotEq {
+ public static class TimeLt extends Lt<Long> {
- private TimeNotEq(long value) {
+ private TimeLt(long value) {
super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- long time = (long) value;
- if (time == Long.MIN_VALUE) {
- return Collections.singletonList(new TimeRange(time + 1, Long.MAX_VALUE));
- } else if (time == Long.MAX_VALUE) {
- return Collections.singletonList(new TimeRange(Long.MIN_VALUE, time - 1));
+ long right = value;
+ if (right != Long.MIN_VALUE) {
+ return Collections.singletonList(new TimeRange(Long.MIN_VALUE, right - 1));
} else {
- return Arrays.asList(
- new TimeRange(Long.MIN_VALUE, time - 1), new TimeRange(time + 1, Long.MAX_VALUE));
+ return Collections.emptyList();
}
}
}
- public static class TimeGt extends Gt {
+ public static class TimeLtEq extends LtEq<Long> {
- private TimeGt(long value) {
+ private TimeLtEq(long value) {
super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- long left = (long) value;
- if (left != Long.MAX_VALUE) {
- return Collections.singletonList(new TimeRange(left + 1, Long.MAX_VALUE));
- } else {
- return Collections.emptyList();
- }
+ return Collections.singletonList(new TimeRange(Long.MIN_VALUE, value));
}
}
- public static class TimeGtEq extends GtEq {
+ public static class TimeEq extends Eq<Long> {
- private TimeGtEq(long value) {
+ private TimeEq(long value) {
super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- return Collections.singletonList(new TimeRange((long) value, Long.MAX_VALUE));
+ return Collections.singletonList(new TimeRange(value, value));
}
}
- public static class TimeLt extends Lt {
+ public static class TimeNotEq extends NotEq<Long> {
- private TimeLt(long value) {
+ private TimeNotEq(long value) {
super(value, FilterType.TIME_FILTER);
}
@Override
public List<TimeRange> getTimeRanges() {
- long right = (long) value;
- if (right != Long.MIN_VALUE) {
- return Collections.singletonList(new TimeRange(Long.MIN_VALUE, right - 1));
+ long time = value;
+ if (time == Long.MIN_VALUE) {
+ return Collections.singletonList(new TimeRange(time + 1, Long.MAX_VALUE));
+ } else if (time == Long.MAX_VALUE) {
+ return Collections.singletonList(new TimeRange(Long.MIN_VALUE, time - 1));
} else {
- return Collections.emptyList();
+ return Arrays.asList(
+ new TimeRange(Long.MIN_VALUE, time - 1), new TimeRange(time + 1, Long.MAX_VALUE));
}
}
}
- public static class TimeLtEq extends LtEq {
+ public static class TimeBetween extends Between<Long> {
- private TimeLtEq(long value) {
- super(value, FilterType.TIME_FILTER);
+ private TimeBetween(long value1, long value2, boolean not) {
+ super(value1, value2, FilterType.TIME_FILTER, not);
}
@Override
public List<TimeRange> getTimeRanges() {
- return Collections.singletonList(new TimeRange(Long.MIN_VALUE, (long) value));
+ long left = value1, right = value2;
+ if (not) {
+ List<TimeRange> res = new ArrayList<>();
+ if (left != Long.MIN_VALUE) {
+ res.add(new TimeRange(Long.MIN_VALUE, left - 1));
+ }
+ if (right != Long.MAX_VALUE) {
+ res.add(new TimeRange(right + 1, Long.MAX_VALUE));
+ }
+ return res;
+ } else {
+ return Collections.singletonList(new TimeRange(left, right));
+ }
}
}
- public static class TimeNotFilter extends NotFilter {
+ public static class TimeIn extends In<Long> {
+
+ private TimeIn(Set<Long> values, boolean not) {
+ super(values, FilterType.TIME_FILTER, not);
+ }
- private TimeNotFilter(Filter filter) {
- super(filter);
+ @Override
+ public List<TimeRange> getTimeRanges() {
+ if (not) {
+ return Collections.singletonList(new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE));
+ } else {
+ return values.stream()
+ .map(
+ l -> {
+ long time = l;
+ return new TimeRange(time, time);
+ })
+ .collect(Collectors.toList());
+ }
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java
index 3dcfbe94f8..79a4bdfd09 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java
@@ -18,8 +18,8 @@
*/
package org.apache.iotdb.tsfile.read.filter;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
+import org.apache.iotdb.tsfile.read.filter.operator.Between;
import org.apache.iotdb.tsfile.read.filter.operator.Eq;
import org.apache.iotdb.tsfile.read.filter.operator.Gt;
import org.apache.iotdb.tsfile.read.filter.operator.GtEq;
@@ -28,7 +28,6 @@ import org.apache.iotdb.tsfile.read.filter.operator.Like;
import org.apache.iotdb.tsfile.read.filter.operator.Lt;
import org.apache.iotdb.tsfile.read.filter.operator.LtEq;
import org.apache.iotdb.tsfile.read.filter.operator.NotEq;
-import org.apache.iotdb.tsfile.read.filter.operator.NotFilter;
import org.apache.iotdb.tsfile.read.filter.operator.Regexp;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
@@ -38,88 +37,60 @@ public class ValueFilter {
private ValueFilter() {}
- public static <T extends Comparable<T>> ValueEq<T> eq(T value) {
- return new ValueEq(value);
- }
-
public static <T extends Comparable<T>> ValueGt<T> gt(T value) {
- return new ValueGt(value);
+ return new ValueGt<>(value);
}
public static <T extends Comparable<T>> ValueGtEq<T> gtEq(T value) {
- return new ValueGtEq(value);
+ return new ValueGtEq<>(value);
}
public static <T extends Comparable<T>> ValueLt<T> lt(T value) {
- return new ValueLt(value);
+ return new ValueLt<>(value);
}
public static <T extends Comparable<T>> ValueLtEq<T> ltEq(T value) {
- return new ValueLtEq(value);
+ return new ValueLtEq<>(value);
}
- public static <T extends Comparable<T>> ValueIn<T> in(Set<T> values, boolean not) {
- return new ValueIn(values, not);
+ public static <T extends Comparable<T>> ValueEq<T> eq(T value) {
+ return new ValueEq<>(value);
}
- public static ValueNotFilter not(Filter filter) {
- return new ValueNotFilter(filter);
+ public static <T extends Comparable<T>> ValueNotEq<T> notEq(T value) {
+ return new ValueNotEq<>(value);
}
- public static <T extends Comparable<T>> ValueNotEq<T> notEq(T value) {
- return new ValueNotEq(value);
+ public static <T extends Comparable<T>> ValueBetween<T> between(T value1, T value2) {
+ return new ValueBetween<>(value1, value2, false);
}
- public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value) {
- return new ValueRegexp(value);
+ public static <T extends Comparable<T>> ValueBetween<T> notBetween(T value1, T value2) {
+ return new ValueBetween<>(value1, value2, true);
}
public static <T extends Comparable<T>> ValueLike<T> like(String value) {
- return new ValueLike(value);
+ return new ValueLike<>(value, false);
}
- public static class ValueIn<T extends Comparable<T>> extends In<T> {
-
- private ValueIn(Set<T> values, boolean not) {
- super(values, FilterType.VALUE_FILTER, not);
- }
+ public static <T extends Comparable<T>> ValueLike<T> notLike(String value) {
+ return new ValueLike<>(value, true);
}
- public static class VectorValueIn<T extends Comparable<T>> extends ValueIn<T> {
-
- private final int index;
-
- private VectorValueIn(Set<T> values, boolean not, int index) {
- super(values, not);
- this.index = index;
- }
-
- public boolean satisfy(long time, TsPrimitiveType[] values) {
- Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue();
- return this.values.contains(v) != not;
- }
+ public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value) {
+ return new ValueRegexp<>(value, false);
}
- public static class ValueEq<T extends Comparable<T>> extends Eq<T> {
-
- private ValueEq(T value) {
- super(value, FilterType.VALUE_FILTER);
- }
+ public static <T extends Comparable<T>> ValueRegexp<T> notRegexp(String value) {
+ return new ValueRegexp<>(value, true);
}
- public static class VectorValueEq<T extends Comparable<T>> extends ValueEq<T> {
-
- private final int index;
-
- private VectorValueEq(T value, int index) {
- super(value);
- this.index = index;
- }
+ public static <T extends Comparable<T>> ValueIn<T> in(Set<T> values) {
+ return new ValueIn<>(values, false);
+ }
- public boolean satisfy(long time, TsPrimitiveType[] values) {
- Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue();
- return this.value.equals(v);
- }
+ public static <T extends Comparable<T>> ValueIn<T> notIn(Set<T> values) {
+ return new ValueIn<>(values, true);
}
public static class ValueGt<T extends Comparable<T>> extends Gt<T> {
@@ -210,15 +181,25 @@ public class ValueFilter {
}
}
- public static class ValueNotFilter extends NotFilter {
+ public static class ValueEq<T extends Comparable<T>> extends Eq<T> {
- private ValueNotFilter(Filter filter) {
- super(filter);
+ private ValueEq(T value) {
+ super(value, FilterType.VALUE_FILTER);
}
+ }
+
+ public static class VectorValueEq<T extends Comparable<T>> extends ValueEq<T> {
- @Override
- public String toString() {
- return FilterType.VALUE_FILTER + super.toString();
+ private final int index;
+
+ private VectorValueEq(T value, int index) {
+ super(value);
+ this.index = index;
+ }
+
+ public boolean satisfy(long time, TsPrimitiveType[] values) {
+ Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue();
+ return this.value.equals(v);
}
}
@@ -244,32 +225,35 @@ public class ValueFilter {
}
}
- public static class ValueRegexp<T extends Comparable<T>> extends Regexp<T> {
+ public static class ValueBetween<T extends Comparable<T>> extends Between<T> {
- private ValueRegexp(String value) {
- super(value, FilterType.VALUE_FILTER);
+ private ValueBetween(T value1, T value2, boolean not) {
+ super(value1, value2, FilterType.VALUE_FILTER, not);
}
}
- public static class VectorValueRegexp<T extends Comparable<T>> extends ValueRegexp<T> {
+ public static class VectorValueBetween<T extends Comparable<T>> extends ValueBetween<T> {
private final int index;
- private VectorValueRegexp(String value, int index) {
- super(value);
+ private VectorValueBetween(T value1, T value2, boolean not, int index) {
+ super(value1, value2, not);
this.index = index;
}
public boolean satisfy(long time, TsPrimitiveType[] values) {
- Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue();
- return this.value.equals(v);
+ if (filterType != FilterType.VALUE_FILTER) {
+ return false;
+ }
+ Object v = values[index].getValue();
+ return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not;
}
}
public static class ValueLike<T extends Comparable<T>> extends Like<T> {
- private ValueLike(String value) {
- super(value, FilterType.VALUE_FILTER);
+ private ValueLike(String value, boolean not) {
+ super(value, FilterType.VALUE_FILTER, not);
}
}
@@ -277,14 +261,63 @@ public class ValueFilter {
private final int index;
- private VectorValueLike(String value, int index) {
- super(value);
+ private VectorValueLike(String value, int index, boolean not) {
+ super(value, not);
+ this.index = index;
+ }
+
+ public boolean satisfy(long time, TsPrimitiveType[] values) {
+ if (filterType != FilterType.VALUE_FILTER) {
+ return false;
+ }
+ Object value = values[index].getValue();
+ return pattern.matcher(value.toString()).find() != not;
+ }
+ }
+
+ public static class ValueRegexp<T extends Comparable<T>> extends Regexp<T> {
+ private ValueRegexp(String value, boolean not) {
+ super(value, FilterType.VALUE_FILTER, not);
+ }
+ }
+
+ public static class VectorValueRegexp<T extends Comparable<T>> extends ValueRegexp<T> {
+
+ private final int index;
+
+ private VectorValueRegexp(String value, int index, boolean not) {
+ super(value, not);
+ this.index = index;
+ }
+
+ public boolean satisfy(long time, TsPrimitiveType[] values) {
+ if (filterType != FilterType.VALUE_FILTER) {
+ return false;
+ }
+ Object value = values[index].getValue();
+ return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find() != not;
+ }
+ }
+
+ public static class ValueIn<T extends Comparable<T>> extends In<T> {
+
+ private ValueIn(Set<T> values, boolean not) {
+ super(values, FilterType.VALUE_FILTER, not);
+ }
+ }
+
+ public static class VectorValueIn<T extends Comparable<T>> extends ValueIn<T> {
+
+ private final int index;
+
+ private VectorValueIn(Set<T> values, boolean not, int index) {
+ super(values, not);
this.index = index;
}
public boolean satisfy(long time, TsPrimitiveType[] values) {
Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue();
- return this.value.equals(v);
+ return this.values.contains(v) != not;
}
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java
index 1cba9c66bd..0d91b58b08 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java
@@ -57,11 +57,6 @@ public abstract class BinaryFilter implements Filter, Serializable {
return right;
}
- @Override
- public String toString() {
- return "( " + left + "," + right + " )";
- }
-
@Override
public abstract Filter copy();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
index dc0d479c83..691cf1030d 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
@@ -32,12 +32,19 @@ import java.util.List;
public interface Filter {
/**
- * To examine whether the statistics is satisfied with the filter.
+ * To examine whether there are data points satisfied with the filter.
*
* @param statistics statistics with min time, max time, min value, max value.
*/
boolean satisfy(Statistics statistics);
+ /**
+ * To examine whether all data points are satisfied with the filter.
+ *
+ * @param statistics statistics with min time, max time, min value, max value.
+ */
+ boolean allSatisfy(Statistics statistics);
+
/**
* To examine whether the single point(with time and value) is satisfied with the filter.
*
@@ -80,4 +87,6 @@ public interface Filter {
default List<TimeRange> getTimeRanges() {
return Collections.emptyList();
}
+
+ Filter reverse();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
index 4dff4e6337..6d647b96c5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.read.filter.operator.Eq;
import org.apache.iotdb.tsfile.read.filter.operator.Gt;
import org.apache.iotdb.tsfile.read.filter.operator.GtEq;
import org.apache.iotdb.tsfile.read.filter.operator.In;
+import org.apache.iotdb.tsfile.read.filter.operator.Like;
import org.apache.iotdb.tsfile.read.filter.operator.Lt;
import org.apache.iotdb.tsfile.read.filter.operator.LtEq;
import org.apache.iotdb.tsfile.read.filter.operator.NotEq;
@@ -82,9 +83,6 @@ public class FilterFactory {
case LTEQ:
filter = new LtEq<>();
break;
- case BETWEEN:
- filter = new Between<>();
- break;
case IN:
filter = new In<>();
break;
@@ -97,6 +95,12 @@ public class FilterFactory {
case REGEXP:
filter = new Regexp<>();
break;
+ case LIKE:
+ filter = new Like<>();
+ break;
+ case BETWEEN:
+ filter = new Between<>();
+ break;
default:
throw new UnsupportedOperationException("Unknown filter type " + id);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
index 6c5120a3f7..1488c70135 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId;
import java.util.ArrayList;
@@ -43,6 +44,11 @@ public class AndFilter extends BinaryFilter {
return left.satisfy(statistics) && right.satisfy(statistics);
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return left.allSatisfy(statistics) && right.allSatisfy(statistics);
+ }
+
@Override
public boolean satisfy(long time, Object value) {
return left.satisfy(time, value) && right.satisfy(time, value);
@@ -110,4 +116,9 @@ public class AndFilter extends BinaryFilter {
return result;
}
+
+ @Override
+ public Filter reverse() {
+ return FilterFactory.or(left.reverse(), right.reverse());
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java
index 0ca994f315..9a0281a772 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java
@@ -28,9 +28,11 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.Objects;
-public class Between<T extends Comparable<T>> implements Filter {
+public class Between<T extends Comparable<T>> implements Filter, Serializable {
private static final long serialVersionUID = -537390606419370764L;
@@ -51,6 +53,66 @@ public class Between<T extends Comparable<T>> implements Filter {
this.not = not;
}
+ @Override
+ public boolean satisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return true;
+ }
+ return not
+ ? (((T) statistics.getMinValue()).compareTo(value1) < 0
+ || ((T) statistics.getMaxValue()).compareTo(value2) > 0)
+ : (((T) statistics.getMaxValue()).compareTo(value1) >= 0
+ && ((T) statistics.getMinValue()).compareTo(value2) <= 0);
+ }
+ }
+
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return not
+ ? (((T) statistics.getMinValue()).compareTo(value2) > 0
+ || ((T) statistics.getMaxValue()).compareTo(value1) < 0)
+ : (((T) statistics.getMinValue()).compareTo(value1) >= 0
+ && ((T) statistics.getMaxValue()).compareTo(value2) <= 0);
+ }
+ }
+
+ @Override
+ public boolean satisfy(long time, Object value) {
+ Object v = filterType == FilterType.TIME_FILTER ? time : value;
+ return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not;
+ }
+
+ @Override
+ public boolean satisfyStartEndTime(long startTime, long endTime) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return not
+ ? (startTime < (Long) value1 || endTime > (Long) value2)
+ : (endTime >= (Long) value1 && startTime <= (Long) value2);
+ } else {
+ return true;
+ }
+ }
+
+ @Override
+ public boolean containStartEndTime(long startTime, long endTime) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return not
+ ? (startTime > (Long) value2 || endTime < (Long) value1)
+ : (startTime >= (Long) value1 && endTime <= (Long) value2);
+ } else {
+ return false;
+ }
+ }
+
@Override
public void serialize(DataOutputStream outputStream) {
try {
@@ -78,64 +140,29 @@ public class Between<T extends Comparable<T>> implements Filter {
}
@Override
- public boolean satisfy(Statistics statistics) {
- if (filterType == FilterType.TIME_FILTER) {
- long time1 = (Long) value1, time2 = (Long) value2;
- if (not) {
- return statistics.getStartTime() < time1 || statistics.getEndTime() > time2;
- } else {
- return statistics.getEndTime() >= time1 || statistics.getStartTime() <= time2;
- }
- } else {
- if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
- return true;
- }
- if (not) {
- return ((T) statistics.getMinValue()).compareTo(value1) < 0
- || ((T) statistics.getMaxValue()).compareTo(value2) > 0;
- } else {
- return ((T) statistics.getMaxValue()).compareTo(value1) >= 0
- && ((T) statistics.getMinValue()).compareTo(value2) <= 0;
- }
- }
- }
-
- @Override
- public boolean satisfy(long time, Object value) {
- Object v = filterType == FilterType.TIME_FILTER ? time : value;
- return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not;
+ public Filter copy() {
+ return new Between<>(value1, value2, filterType, not);
}
@Override
- public boolean satisfyStartEndTime(long startTime, long endTime) {
- if (filterType == FilterType.TIME_FILTER) {
- long time1 = (Long) value1, time2 = (Long) value2;
- if (not) {
- return startTime < time1 || endTime > time2;
- } else {
- return endTime >= time1 && startTime <= time2;
- }
- } else {
- return true;
+ public boolean equals(Object o) {
+ if (!(o instanceof Between)) {
+ return false;
}
+ Between<?> between = (Between<?>) o;
+ return not == between.not
+ && value1.equals(between.value1)
+ && value2.equals(between.value2)
+ && filterType == between.filterType;
}
@Override
- public boolean containStartEndTime(long startTime, long endTime) {
- if (filterType == FilterType.TIME_FILTER) {
- long time1 = (Long) value1, time2 = (Long) value2;
- if (not) {
- return endTime < time1 || startTime > time2;
- } else {
- return startTime >= time1 && endTime <= time2;
- }
- } else {
- return true;
- }
+ public int hashCode() {
+ return Objects.hash(value1, value2, not, filterType);
}
@Override
- public Filter copy() {
- return new Between(value1, value2, filterType, not);
+ public Filter reverse() {
+ return new Between<>(value1, value2, filterType, !not);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
index 7a69be9468..922a102c9c 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java
@@ -43,8 +43,7 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) >= statistics.getStartTime()
- && ((Long) value) <= statistics.getEndTime();
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -54,6 +53,19 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMinValue()) == 0
+ && value.compareTo((T) statistics.getMaxValue()) == 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -76,13 +88,13 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return time == startTime && time == endTime;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new Eq(value, filterType);
+ return new Eq<>(value, filterType);
}
@Override
@@ -94,4 +106,9 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.EQ;
}
+
+ @Override
+ public Filter reverse() {
+ return new NotEq<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
index 0719397e8c..ffb6c75eab 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java
@@ -43,7 +43,7 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) < statistics.getEndTime();
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -52,6 +52,18 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMinValue()) < 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -74,13 +86,13 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return startTime > time;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new Gt(value, filterType);
+ return new Gt<>(value, filterType);
}
@Override
@@ -92,4 +104,9 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.GT;
}
+
+ @Override
+ public Filter reverse() {
+ return new LtEq<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
index 4e811e14fe..f6b3a2cd86 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java
@@ -43,7 +43,7 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) <= statistics.getEndTime();
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -52,6 +52,18 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMinValue()) <= 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -74,13 +86,13 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return startTime >= time;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new GtEq(value, filterType);
+ return new GtEq<>(value, filterType);
}
@Override
@@ -92,4 +104,9 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.GTEQ;
}
+
+ @Override
+ public Filter reverse() {
+ return new Lt<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
index 60e0747f5d..60d6eed75f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java
@@ -26,11 +26,13 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
/**
@@ -38,7 +40,7 @@ import java.util.Set;
*
* @param <T> comparable data type
*/
-public class In<T extends Comparable<T>> implements Filter {
+public class In<T extends Comparable<T>> implements Filter, Serializable {
private static final long serialVersionUID = 8572705136773595399L;
@@ -61,6 +63,11 @@ public class In<T extends Comparable<T>> implements Filter {
return true;
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return false;
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -74,12 +81,12 @@ public class In<T extends Comparable<T>> implements Filter {
@Override
public boolean containStartEndTime(long startTime, long endTime) {
- return true;
+ return false;
}
@Override
public Filter copy() {
- return new In(new HashSet(values), filterType, not);
+ return new In<>(new HashSet<>(values), filterType, not);
}
@Override
@@ -110,17 +117,23 @@ public class In<T extends Comparable<T>> implements Filter {
@Override
public boolean equals(Object o) {
- return o instanceof In
- && ((In<?>) o).filterType == filterType
- && ((In<?>) o).values.equals(values)
- && ((In<?>) o).not == not;
+ if (!(o instanceof In)) {
+ return false;
+ }
+ In<?> in = (In<?>) o;
+ return in.filterType == filterType && in.values.equals(values) && in.not == not;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(values, not, filterType);
}
@Override
public String toString() {
List<T> valueList = new ArrayList<>(values);
Collections.sort(valueList);
- return filterType + " < " + "reverse: " + not + ", " + valueList;
+ return filterType + (not ? " not in " : " in ") + valueList;
}
@Override
@@ -128,6 +141,11 @@ public class In<T extends Comparable<T>> implements Filter {
return FilterSerializeId.IN;
}
+ @Override
+ public Filter reverse() {
+ return new In<>(new HashSet<>(values), filterType, !not);
+ }
+
public Set<T> getValues() {
return values;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java
index fac3dfcf5b..d509c00862 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java
@@ -26,7 +26,9 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.Objects;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -35,7 +37,7 @@ import java.util.regex.PatternSyntaxException;
*
* @param <T> comparable data type
*/
-public class Like<T extends Comparable<T>> implements Filter {
+public class Like<T extends Comparable<T>> implements Filter, Serializable {
private static final long serialVersionUID = 2171102599229260789L;
@@ -45,15 +47,18 @@ public class Like<T extends Comparable<T>> implements Filter {
protected Pattern pattern;
- private Like() {}
+ protected boolean not;
+
+ public Like() {}
/**
* The main idea of this part comes from
* https://codereview.stackexchange.com/questions/36861/convert-sql-like-to-regex/36864
*/
- public Like(String value, FilterType filterType) {
+ public Like(String value, FilterType filterType, boolean not) {
this.value = value;
this.filterType = filterType;
+ this.not = not;
try {
String unescapeValue = unescapeString(value);
String specialRegexStr = ".^$*+?{}[]|()";
@@ -89,12 +94,17 @@ public class Like<T extends Comparable<T>> implements Filter {
return true;
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return false;
+ }
+
@Override
public boolean satisfy(long time, Object value) {
if (filterType != FilterType.VALUE_FILTER) {
- return false;
+ throw new UnsupportedOperationException("");
}
- return pattern.matcher(value.toString()).find();
+ return not != pattern.matcher(value.toString()).find();
}
@Override
@@ -104,12 +114,12 @@ public class Like<T extends Comparable<T>> implements Filter {
@Override
public boolean containStartEndTime(long startTime, long endTime) {
- return true;
+ return false;
}
@Override
public Filter copy() {
- return new Like(value, filterType);
+ return new Like<>(value, filterType, not);
}
@Override
@@ -117,9 +127,10 @@ public class Like<T extends Comparable<T>> implements Filter {
try {
outputStream.write(getSerializeId().ordinal());
outputStream.write(filterType.ordinal());
- ReadWriteIOUtils.writeObject(value, outputStream);
- } catch (IOException ex) {
- throw new IllegalArgumentException("Failed to serialize outputStream of type:", ex);
+ ReadWriteIOUtils.write(value, outputStream);
+ ReadWriteIOUtils.write(not, outputStream);
+ } catch (IOException ignored) {
+ // ignore
}
}
@@ -127,11 +138,12 @@ public class Like<T extends Comparable<T>> implements Filter {
public void deserialize(ByteBuffer buffer) {
filterType = FilterType.values()[buffer.get()];
value = ReadWriteIOUtils.readString(buffer);
+ not = ReadWriteIOUtils.readBool(buffer);
}
@Override
public String toString() {
- return filterType + " is " + value;
+ return filterType + (not ? " not like " : " like ") + value;
}
@Override
@@ -139,11 +151,30 @@ public class Like<T extends Comparable<T>> implements Filter {
return FilterSerializeId.LIKE;
}
+ @Override
+ public Filter reverse() {
+ return new Like<>(value, filterType, !not);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Like)) {
+ return false;
+ }
+ Like<?> like = (Like<?>) o;
+ return not == like.not && value.equals(like.value) && filterType == like.filterType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value, filterType, not);
+ }
+
/**
* This Method is for unescaping strings except '\' before special string '%', '_', '\', because
* we need to use '\' to judege whether to replace this to regexp string
*/
- public String unescapeString(String value) {
+ private String unescapeString(String value) {
String out = "";
for (int i = 0; i < value.length(); i++) {
String ch = String.valueOf(value.charAt(i));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
index 9c83b720c2..c0151d3f63 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java
@@ -43,7 +43,7 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) > statistics.getStartTime();
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -52,6 +52,18 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMaxValue()) > 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -74,13 +86,13 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return endTime < time;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new Lt(value, filterType);
+ return new Lt<>(value, filterType);
}
@Override
@@ -92,4 +104,9 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.LT;
}
+
+ @Override
+ public Filter reverse() {
+ return new GtEq<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
index b68b1b4759..4b35f154a1 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java
@@ -43,7 +43,7 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return ((Long) value) >= statistics.getStartTime();
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -52,6 +52,18 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMaxValue()) >= 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -74,13 +86,13 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return endTime <= time;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new LtEq(value, filterType);
+ return new LtEq<>(value, filterType);
}
@Override
@@ -92,4 +104,9 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.LTEQ;
}
+
+ @Override
+ public Filter reverse() {
+ return new Gt<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
index 55abda179e..45148c5c03 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java
@@ -43,8 +43,7 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> {
@Override
public boolean satisfy(Statistics statistics) {
if (filterType == FilterType.TIME_FILTER) {
- return !(((Long) value) == statistics.getStartTime()
- && (Long) value == statistics.getEndTime());
+ return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime());
} else {
if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
return true;
@@ -54,6 +53,19 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> {
}
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ if (filterType == FilterType.TIME_FILTER) {
+ return containStartEndTime(statistics.getStartTime(), statistics.getEndTime());
+ } else {
+ if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) {
+ return false;
+ }
+ return value.compareTo((T) statistics.getMinValue()) < 0
+ || value.compareTo((T) statistics.getMaxValue()) > 0;
+ }
+ }
+
@Override
public boolean satisfy(long time, Object value) {
Object v = filterType == FilterType.TIME_FILTER ? time : value;
@@ -76,13 +88,13 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> {
long time = (Long) value;
return time < startTime || time > endTime;
} else {
- return true;
+ return false;
}
}
@Override
public Filter copy() {
- return new NotEq(value, filterType);
+ return new NotEq<>(value, filterType);
}
@Override
@@ -94,4 +106,9 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> {
public FilterSerializeId getSerializeId() {
return FilterSerializeId.NEQ;
}
+
+ @Override
+ public Filter reverse() {
+ return new Eq<>(value, filterType);
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
index cd983dc329..ef21c14a0c 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java
@@ -32,12 +32,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
-/** NotFilter necessary. Use InvertExpressionVisitor */
public class NotFilter implements Filter, Serializable {
private static final long serialVersionUID = 584860326604020881L;
private Filter that;
+ public static final String CONTAIN_NOT_ERR_MSG =
+ "This predicate contains a not! Did you forget to run this predicate through PredicateRemoveNotRewriter? ";
+
public NotFilter() {}
public NotFilter(Filter that) {
@@ -46,7 +48,12 @@ public class NotFilter implements Filter, Serializable {
@Override
public boolean satisfy(Statistics statistics) {
- return !that.satisfy(statistics);
+ throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this);
+ }
+
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this);
}
@Override
@@ -54,18 +61,14 @@ public class NotFilter implements Filter, Serializable {
return !that.satisfy(time, value);
}
- /**
- * Notice that, if the not filter only contains value filter, this method may return false, this
- * may cause misunderstanding.
- */
@Override
public boolean satisfyStartEndTime(long startTime, long endTime) {
- return !that.satisfyStartEndTime(startTime, endTime);
+ throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this);
}
@Override
public boolean containStartEndTime(long startTime, long endTime) {
- return !that.satisfyStartEndTime(startTime, endTime);
+ throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this);
}
@Override
@@ -79,7 +82,7 @@ public class NotFilter implements Filter, Serializable {
@Override
public String toString() {
- return "NotFilter: " + that;
+ return "not (" + that + ")";
}
@Override
@@ -139,4 +142,9 @@ public class NotFilter implements Filter, Serializable {
}
return res;
}
+
+ @Override
+ public Filter reverse() {
+ return that;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
index 17b0935510..0a16909e2c 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId;
import java.io.Serializable;
@@ -55,6 +56,11 @@ public class OrFilter extends BinaryFilter implements Serializable {
return left.satisfy(statistics) || right.satisfy(statistics);
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return left.allSatisfy(statistics) || right.allSatisfy(statistics);
+ }
+
@Override
public boolean satisfy(long time, Object value) {
return left.satisfy(time, value) || right.satisfy(time, value);
@@ -138,4 +144,9 @@ public class OrFilter extends BinaryFilter implements Serializable {
return result;
}
+
+ @Override
+ public Filter reverse() {
+ return FilterFactory.and(left.reverse(), right.reverse());
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
index 31e7260469..3a154e7320 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.regex.Pattern;
@@ -37,7 +38,9 @@ import java.util.regex.PatternSyntaxException;
*
* @param <T> comparable data type
*/
-public class Regexp<T extends Comparable<T>> implements Filter {
+public class Regexp<T extends Comparable<T>> implements Filter, Serializable {
+
+ private static final long serialVersionUID = -1168073851950524983L;
protected String value;
@@ -45,11 +48,14 @@ public class Regexp<T extends Comparable<T>> implements Filter {
protected Pattern pattern;
+ protected boolean not;
+
public Regexp() {}
- public Regexp(String value, FilterType filterType) {
+ public Regexp(String value, FilterType filterType, boolean not) {
this.value = value;
this.filterType = filterType;
+ this.not = not;
try {
this.pattern = Pattern.compile(this.value);
} catch (PatternSyntaxException e) {
@@ -62,12 +68,17 @@ public class Regexp<T extends Comparable<T>> implements Filter {
return true;
}
+ @Override
+ public boolean allSatisfy(Statistics statistics) {
+ return false;
+ }
+
@Override
public boolean satisfy(long time, Object value) {
if (filterType != FilterType.VALUE_FILTER) {
- return false;
+ throw new UnsupportedOperationException("");
}
- return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
+ return not != pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
}
@Override
@@ -77,12 +88,12 @@ public class Regexp<T extends Comparable<T>> implements Filter {
@Override
public boolean containStartEndTime(long startTime, long endTime) {
- return true;
+ return false;
}
@Override
public Filter copy() {
- return new Regexp(value, filterType);
+ return new Regexp<>(value, filterType, not);
}
@Override
@@ -91,8 +102,9 @@ public class Regexp<T extends Comparable<T>> implements Filter {
outputStream.write(getSerializeId().ordinal());
outputStream.write(filterType.ordinal());
ReadWriteIOUtils.write(value, outputStream);
- } catch (IOException ex) {
- throw new IllegalArgumentException("Failed to serialize outputStream of type:", ex);
+ ReadWriteIOUtils.write(not, outputStream);
+ } catch (IOException ignored) {
+ // ignore
}
}
@@ -100,6 +112,7 @@ public class Regexp<T extends Comparable<T>> implements Filter {
public void deserialize(ByteBuffer buffer) {
filterType = FilterType.values()[buffer.get()];
value = ReadWriteIOUtils.readString(buffer);
+ not = ReadWriteIOUtils.readBool(buffer);
if (value != null) {
try {
this.pattern = Pattern.compile(value);
@@ -111,22 +124,34 @@ public class Regexp<T extends Comparable<T>> implements Filter {
@Override
public String toString() {
- return filterType + " is " + value;
+ return filterType + (not ? " not match " : " match ") + value;
+ }
+
+ @Override
+ public FilterSerializeId getSerializeId() {
+ return FilterSerializeId.REGEXP;
+ }
+
+ @Override
+ public Filter reverse() {
+ return new Regexp<>(value, filterType, !not);
}
@Override
public boolean equals(Object o) {
- return o instanceof Regexp
- && Objects.equals(((Regexp<?>) o).value, value)
- && ((Regexp<?>) o).filterType == filterType;
+ if (!(o instanceof Regexp)) {
+ return false;
+ }
+ Regexp<?> regexp = (Regexp<?>) o;
+ return not == regexp.not && value.equals(regexp.value) && filterType == regexp.filterType;
}
@Override
- public FilterSerializeId getSerializeId() {
- return FilterSerializeId.REGEXP;
+ public int hashCode() {
+ return Objects.hash(value, filterType, not);
}
- private static class AccessCount {
+ public static class AccessCount {
private int count;
private final int accessThreshold =
TSFileDescriptor.getInstance().getConfig().getPatternMatchingThreshold();
@@ -138,7 +163,7 @@ public class Regexp<T extends Comparable<T>> implements Filter {
}
}
- private static class MatcherInput implements CharSequence {
+ public static class MatcherInput implements CharSequence {
private final CharSequence value;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
index 795116ccb5..60b74a1870 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java
@@ -163,16 +163,18 @@ public class PageReader implements IPageReader {
}
private boolean pageSatisfy() {
- if (filter != null) {
- return filter.satisfy(getStatistics());
- } else {
- long rowCount = getStatistics().getCount();
+ Statistics statistics = getStatistics();
+ if (filter == null || filter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
if (paginationController.hasCurOffset(rowCount)) {
paginationController.consumeOffset(rowCount);
return false;
+ } else {
+ return true;
}
+ } else {
+ return filter.satisfy(statistics);
}
- return true;
}
@Override
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
index 779e32d404..2d3b5c0abf 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java
@@ -43,12 +43,17 @@ public class FilterSerializeTest {
ValueFilter.gtEq("filter"),
ValueFilter.lt(0.1),
ValueFilter.ltEq(0.01f),
- ValueFilter.not(ValueFilter.eq(true)),
+ FilterFactory.not(ValueFilter.eq(true)),
ValueFilter.notEq(false),
ValueFilter.notEq(false),
- ValueFilter.in(new HashSet<>(Arrays.asList("a", "b")), false),
- ValueFilter.in(new HashSet<>(Arrays.asList("c", "d")), true),
+ ValueFilter.in(new HashSet<>(Arrays.asList("a", "b"))),
+ ValueFilter.notIn(new HashSet<>(Arrays.asList("c", "d"))),
ValueFilter.regexp("s.*"),
+ ValueFilter.like("s.*"),
+ ValueFilter.notRegexp("s.*"),
+ ValueFilter.notLike("s.*"),
+ ValueFilter.between(1, 100),
+ ValueFilter.notBetween(1, 100)
};
for (Filter filter : filters) {
validateSerialization(filter);
@@ -64,11 +69,13 @@ public class FilterSerializeTest {
TimeFilter.gtEq(3),
TimeFilter.lt(4),
TimeFilter.ltEq(5),
- TimeFilter.not(ValueFilter.eq(6)),
+ FilterFactory.not(ValueFilter.eq(6)),
TimeFilter.notEq(7),
TimeFilter.notEq(7),
- TimeFilter.in(new HashSet<>(Arrays.asList(1L, 2L)), false),
- TimeFilter.in(new HashSet<>(Arrays.asList(3L, 4L)), true),
+ TimeFilter.in(new HashSet<>(Arrays.asList(1L, 2L))),
+ TimeFilter.notIn(new HashSet<>(Arrays.asList(3L, 4L))),
+ TimeFilter.between(1, 100),
+ TimeFilter.notBetween(1, 100)
};
for (Filter filter : filters) {
validateSerialization(filter);
@@ -80,7 +87,7 @@ public class FilterSerializeTest {
Filter[] filters =
new Filter[] {
FilterFactory.and(TimeFilter.eq(1), ValueFilter.eq(1)),
- FilterFactory.or(ValueFilter.gt(2L), TimeFilter.not(ValueFilter.eq(6)))
+ FilterFactory.or(ValueFilter.gt(2L), FilterFactory.not(ValueFilter.eq(6)))
};
for (Filter filter : filters) {
validateSerialization(filter);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java
index df4f9d06f3..cf1b31a9ef 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java
@@ -25,6 +25,9 @@ import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
import org.junit.Assert;
import org.junit.Test;
+import static org.apache.iotdb.tsfile.read.filter.operator.NotFilter.CONTAIN_NOT_ERR_MSG;
+import static org.junit.Assert.fail;
+
public class MinTimeMaxTimeFilterTest {
long minTime = 100;
@@ -81,7 +84,7 @@ public class MinTimeMaxTimeFilterTest {
Filter valueEq = ValueFilter.gt(100);
Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime));
+ Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime));
}
@Test
@@ -104,11 +107,11 @@ public class MinTimeMaxTimeFilterTest {
Filter valueEq = ValueFilter.gtEq(100);
Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime));
+ Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime));
valueEq = ValueFilter.gtEq(150);
Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime));
+ Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime));
}
@Test
@@ -131,7 +134,7 @@ public class MinTimeMaxTimeFilterTest {
Filter valueEq = ValueFilter.lt(100);
Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime));
+ Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime));
}
@Test
@@ -154,7 +157,7 @@ public class MinTimeMaxTimeFilterTest {
Filter valueEq = ValueFilter.ltEq(100);
Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime));
+ Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime));
}
@Test
@@ -193,22 +196,17 @@ public class MinTimeMaxTimeFilterTest {
@Test
public void testNot() {
Filter not = FilterFactory.not(TimeFilter.ltEq(10L));
- Assert.assertTrue(not.satisfyStartEndTime(minTime, maxTime));
- Assert.assertTrue(not.containStartEndTime(minTime, maxTime));
-
- not = FilterFactory.not(TimeFilter.ltEq(100L));
- Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime));
- Assert.assertFalse(not.containStartEndTime(minTime, maxTime));
-
- not = FilterFactory.not(TimeFilter.ltEq(200L));
- Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime));
- Assert.assertFalse(not.containStartEndTime(minTime, maxTime));
-
- not = FilterFactory.not(TimeFilter.ltEq(300L));
- Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime));
- Assert.assertFalse(not.containStartEndTime(minTime, maxTime));
-
- not = FilterFactory.not(ValueFilter.ltEq(100));
- Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime));
+ try {
+ not.satisfyStartEndTime(minTime, maxTime);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
+ try {
+ not.containStartEndTime(minTime, maxTime);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java
index 7caec9eaca..12c8162966 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java
@@ -103,12 +103,12 @@ public class OperatorTest {
@Test
public void testNot() {
- Filter timeLt = TimeFilter.not(TimeFilter.lt(TESTED_TIMESTAMP));
+ Filter timeLt = FilterFactory.not(TimeFilter.lt(TESTED_TIMESTAMP));
Assert.assertFalse(timeLt.satisfy(TESTED_TIMESTAMP - 1, 100));
Assert.assertTrue(timeLt.satisfy(TESTED_TIMESTAMP, 100));
Assert.assertTrue(timeLt.satisfy(TESTED_TIMESTAMP + 1, 100));
- Filter valueLt = ValueFilter.not(ValueFilter.lt(100L));
+ Filter valueLt = FilterFactory.not(ValueFilter.lt(100L));
Assert.assertFalse(valueLt.satisfy(TESTED_TIMESTAMP, 99L));
Assert.assertTrue(valueLt.satisfy(TESTED_TIMESTAMP, 100L));
Assert.assertTrue(valueLt.satisfy(TESTED_TIMESTAMP, 101L));
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java
new file mode 100644
index 0000000000..f436a52238
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.read.filter;
+
+import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
+import org.apache.iotdb.tsfile.read.filter.factory.FilterType;
+import org.apache.iotdb.tsfile.read.filter.operator.Between;
+import org.apache.iotdb.tsfile.read.filter.operator.Eq;
+import org.apache.iotdb.tsfile.read.filter.operator.Gt;
+import org.apache.iotdb.tsfile.read.filter.operator.GtEq;
+import org.apache.iotdb.tsfile.read.filter.operator.In;
+import org.apache.iotdb.tsfile.read.filter.operator.Like;
+import org.apache.iotdb.tsfile.read.filter.operator.Lt;
+import org.apache.iotdb.tsfile.read.filter.operator.LtEq;
+import org.apache.iotdb.tsfile.read.filter.operator.NotEq;
+import org.apache.iotdb.tsfile.read.filter.operator.Regexp;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+public class PredicateRemoveNotRewriterTest {
+
+ @Test
+ public void testReverse() {
+ Assert.assertEquals(
+ new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new GtEq<>(1, FilterType.TIME_FILTER), new Lt<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new Lt<>(1, FilterType.TIME_FILTER), new GtEq<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new Eq<>(1, FilterType.TIME_FILTER), new NotEq<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new NotEq<>(1, FilterType.TIME_FILTER), new Eq<>(1, FilterType.TIME_FILTER).reverse());
+ Assert.assertEquals(
+ new Like<>("s*", FilterType.TIME_FILTER, true),
+ new Like<>("s*", FilterType.TIME_FILTER, false).reverse());
+ Assert.assertEquals(
+ new Like<>("s*", FilterType.TIME_FILTER, false),
+ new Like<>("s*", FilterType.TIME_FILTER, true).reverse());
+ Assert.assertEquals(
+ new Regexp<>("s*", FilterType.TIME_FILTER, true),
+ new Regexp<>("s*", FilterType.TIME_FILTER, false).reverse());
+ Assert.assertEquals(
+ new Regexp<>("s*", FilterType.TIME_FILTER, false),
+ new Regexp<>("s*", FilterType.TIME_FILTER, true).reverse());
+ Assert.assertEquals(
+ new Between<>(1, 100, FilterType.TIME_FILTER, true),
+ new Between<>(1, 100, FilterType.TIME_FILTER, false).reverse());
+ Assert.assertEquals(
+ new Between<>(1, 100, FilterType.TIME_FILTER, false),
+ new Between<>(1, 100, FilterType.TIME_FILTER, true).reverse());
+ Assert.assertEquals(
+ new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, true),
+ new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, false).reverse());
+ Assert.assertEquals(
+ new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, false),
+ new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, true).reverse());
+ Assert.assertEquals(
+ new Gt<>(1, FilterType.TIME_FILTER),
+ FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER)).reverse());
+ Assert.assertEquals(
+ FilterFactory.and(
+ new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)),
+ FilterFactory.or(new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER))
+ .reverse());
+ Assert.assertEquals(
+ FilterFactory.or(
+ new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER)),
+ FilterFactory.and(
+ new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER))
+ .reverse());
+ }
+
+ @Test
+ public void testRemoveNot() {
+ Assert.assertEquals(
+ new LtEq<>(1, FilterType.TIME_FILTER),
+ PredicateRemoveNotRewriter.rewrite(FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER))));
+ Assert.assertEquals(
+ new Like<>("s*", FilterType.TIME_FILTER, false),
+ PredicateRemoveNotRewriter.rewrite(
+ FilterFactory.not(new Like<>("s*", FilterType.TIME_FILTER, true))));
+ Assert.assertEquals(
+ FilterFactory.or(
+ new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)),
+ PredicateRemoveNotRewriter.rewrite(
+ FilterFactory.or(
+ FilterFactory.not(new LtEq<>(1, FilterType.TIME_FILTER)),
+ FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER)))));
+ Assert.assertEquals(
+ FilterFactory.and(
+ new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)),
+ PredicateRemoveNotRewriter.rewrite(
+ FilterFactory.and(
+ FilterFactory.not(new LtEq<>(1, FilterType.TIME_FILTER)),
+ FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER)))));
+ }
+}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java
index 33e3f958da..d50b44a7a6 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java
@@ -27,10 +27,19 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.Serializable;
+
+import static org.apache.iotdb.tsfile.read.filter.operator.NotFilter.CONTAIN_NOT_ERR_MSG;
+import static org.junit.Assert.fail;
+
public class StatisticsFilterTest {
- private Statistics statistics1 = Statistics.getStatsByType(TSDataType.INT64);
- private Statistics statistics2 = Statistics.getStatsByType(TSDataType.INT64);
+ private final Statistics<? extends Serializable> statistics1 =
+ Statistics.getStatsByType(TSDataType.INT64);
+ private final Statistics<? extends Serializable> statistics2 =
+ Statistics.getStatsByType(TSDataType.INT64);
+ private final Statistics<? extends Serializable> statistics3 =
+ Statistics.getStatsByType(TSDataType.INT64);
@Before
public void before() {
@@ -38,6 +47,7 @@ public class StatisticsFilterTest {
statistics1.update(100L, 100L);
statistics2.update(101L, 101L);
statistics2.update(200L, 200L);
+ statistics3.update(10L, 10L);
}
@Test
@@ -45,32 +55,63 @@ public class StatisticsFilterTest {
Filter timeEq = TimeFilter.eq(10L);
Assert.assertTrue(timeEq.satisfy(statistics1));
Assert.assertFalse(timeEq.satisfy(statistics2));
+ Assert.assertFalse(timeEq.allSatisfy(statistics1));
+ Assert.assertFalse(timeEq.allSatisfy(statistics2));
+ Assert.assertTrue(timeEq.allSatisfy(statistics3));
Filter valueEq = ValueFilter.eq(101L);
Assert.assertFalse(valueEq.satisfy(statistics1));
Assert.assertTrue(valueEq.satisfy(statistics2));
+ Assert.assertFalse(valueEq.allSatisfy(statistics1));
+ Assert.assertFalse(valueEq.allSatisfy(statistics2));
+ Assert.assertFalse(valueEq.allSatisfy(statistics3));
+ }
+
+ @Test
+ public void testNotEq() {
+ Filter timeNotEq = TimeFilter.notEq(10L);
+ Assert.assertTrue(timeNotEq.satisfy(statistics1));
+ Assert.assertTrue(timeNotEq.satisfy(statistics2));
+ Assert.assertFalse(timeNotEq.allSatisfy(statistics1));
+ Assert.assertTrue(timeNotEq.allSatisfy(statistics2));
+ Assert.assertFalse(timeNotEq.allSatisfy(statistics3));
+
+ Filter valueNotEq = ValueFilter.notEq(101L);
+ Assert.assertTrue(valueNotEq.satisfy(statistics1));
+ Assert.assertTrue(valueNotEq.satisfy(statistics2));
+ Assert.assertTrue(valueNotEq.allSatisfy(statistics1));
+ Assert.assertFalse(valueNotEq.allSatisfy(statistics2));
+ Assert.assertTrue(valueNotEq.allSatisfy(statistics3));
}
@Test
public void testGt() {
- Filter timeGt = TimeFilter.gt(100L);
- Assert.assertFalse(timeGt.satisfy(statistics1));
+ Filter timeGt = TimeFilter.gt(10L);
+ Assert.assertTrue(timeGt.satisfy(statistics1));
Assert.assertTrue(timeGt.satisfy(statistics2));
+ Assert.assertFalse(timeGt.allSatisfy(statistics1));
+ Assert.assertTrue(timeGt.allSatisfy(statistics2));
Filter valueGt = ValueFilter.gt(100L);
Assert.assertFalse(valueGt.satisfy(statistics1));
Assert.assertTrue(valueGt.satisfy(statistics2));
+ Assert.assertFalse(valueGt.allSatisfy(statistics1));
+ Assert.assertTrue(valueGt.allSatisfy(statistics2));
}
@Test
public void testGtEq() {
- Filter timeGtEq = TimeFilter.gtEq(100L);
+ Filter timeGtEq = TimeFilter.gtEq(10L);
Assert.assertTrue(timeGtEq.satisfy(statistics1));
Assert.assertTrue(timeGtEq.satisfy(statistics2));
+ Assert.assertFalse(timeGtEq.allSatisfy(statistics1));
+ Assert.assertTrue(timeGtEq.allSatisfy(statistics2));
Filter valueGtEq = ValueFilter.gtEq(100L);
Assert.assertTrue(valueGtEq.satisfy(statistics1));
Assert.assertTrue(valueGtEq.satisfy(statistics2));
+ Assert.assertFalse(valueGtEq.allSatisfy(statistics1));
+ Assert.assertTrue(valueGtEq.allSatisfy(statistics2));
}
@Test
@@ -78,10 +119,14 @@ public class StatisticsFilterTest {
Filter timeLt = TimeFilter.lt(101L);
Assert.assertTrue(timeLt.satisfy(statistics1));
Assert.assertFalse(timeLt.satisfy(statistics2));
+ Assert.assertTrue(timeLt.allSatisfy(statistics1));
+ Assert.assertFalse(timeLt.allSatisfy(statistics2));
- Filter valueLt = ValueFilter.lt(101L);
+ Filter valueLt = ValueFilter.lt(11L);
Assert.assertTrue(valueLt.satisfy(statistics1));
Assert.assertFalse(valueLt.satisfy(statistics2));
+ Assert.assertFalse(valueLt.allSatisfy(statistics1));
+ Assert.assertFalse(valueLt.allSatisfy(statistics2));
}
@Test
@@ -90,9 +135,11 @@ public class StatisticsFilterTest {
Assert.assertTrue(timeLtEq.satisfy(statistics1));
Assert.assertTrue(timeLtEq.satisfy(statistics2));
- Filter valueLtEq = ValueFilter.ltEq(101L);
+ Filter valueLtEq = ValueFilter.ltEq(11L);
Assert.assertTrue(valueLtEq.satisfy(statistics1));
- Assert.assertTrue(valueLtEq.satisfy(statistics2));
+ Assert.assertFalse(valueLtEq.satisfy(statistics2));
+ Assert.assertFalse(valueLtEq.allSatisfy(statistics1));
+ Assert.assertFalse(valueLtEq.allSatisfy(statistics2));
}
@Test
@@ -105,4 +152,54 @@ public class StatisticsFilterTest {
Assert.assertTrue(orFilter.satisfy(statistics1));
Assert.assertTrue(orFilter.satisfy(statistics2));
}
+
+ @Test
+ public void testNot() {
+ Filter timeNotEq = FilterFactory.not(TimeFilter.eq(10L));
+ try {
+ timeNotEq.satisfy(statistics1);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
+ try {
+ timeNotEq.allSatisfy(statistics1);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
+
+ Filter valueNotEq = FilterFactory.not(ValueFilter.eq(101L));
+ try {
+ valueNotEq.satisfy(statistics1);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
+ try {
+ valueNotEq.allSatisfy(statistics1);
+ fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG));
+ }
+ }
+
+ @Test
+ public void testBetweenAnd() {
+ Filter timeBetweenAnd = TimeFilter.between(0, 20);
+ Assert.assertTrue(timeBetweenAnd.satisfy(statistics1));
+ Assert.assertFalse(timeBetweenAnd.satisfy(statistics2));
+ Assert.assertTrue(timeBetweenAnd.satisfy(statistics3));
+ Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics1));
+ Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics2));
+ Assert.assertTrue(timeBetweenAnd.allSatisfy(statistics3));
+
+ Filter timeNotBetweenAnd = TimeFilter.notBetween(0, 20);
+ Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics1));
+ Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics2));
+ Assert.assertFalse(timeNotBetweenAnd.satisfy(statistics3));
+ Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics1));
+ Assert.assertTrue(timeNotBetweenAnd.allSatisfy(statistics2));
+ Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics3));
+ }
}