You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/08/15 01:51:43 UTC
[iotdb] branch master updated: [IOTDB-6112] Fix Limit & Offset push down doesn't take effect while there exist time filter
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new cdcb4a691d3 [IOTDB-6112] Fix Limit & Offset push down doesn't take effect while there exist time filter
cdcb4a691d3 is described below
commit cdcb4a691d3563f40904db3f2d9b314d573d9c55
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Tue Aug 15 09:51:38 2023 +0800
[IOTDB-6112] Fix Limit & Offset push down doesn't take effect while there exist time filter
---
.../operator/source/AlignedSeriesScanUtil.java | 18 ++++++++----------
.../read/reader/chunk/MemAlignedPageReader.java | 14 ++++++++------
.../tsfile/read/reader/page/AlignedPageReader.java | 16 +++++++++-------
3 files changed, 25 insertions(+), 23 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
index 360211805bb..d36de2e4bf7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
@@ -141,12 +141,11 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil {
&& !isFileOverlapped()
&& !firstTimeSeriesMetadata.isModified()) {
Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) {
- skipCurrentFile();
- }
- } else {
+ Statistics statistics = firstTimeSeriesMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
skipOffsetByTimeSeriesMetadata();
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentFile();
}
}
}
@@ -178,12 +177,11 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil {
protected void filterFirstChunkMetadata() throws IOException {
if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) {
Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstChunkMetadata.getStatistics())) {
- skipCurrentChunk();
- }
- } else {
+ Statistics statistics = firstChunkMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
skipOffsetByChunkMetadata();
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentChunk();
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
index 6bce32c0a08..b1bba47f129 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
@@ -99,17 +99,16 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader {
}
private boolean pageSatisfy() {
- if (valueFilter != null) {
- return valueFilter.satisfy(getStatistics());
- } else {
+ Statistics<? extends Serializable> statistics = getStatistics();
+ if (valueFilter == null || valueFilter.allSatisfy(statistics)) {
// For aligned series, When we only read some measurements under an aligned device, if the
// values of these queried measurements at a timestamp are all null, the timestamp will not be
// selected.
// NOTE: if we change the read semantic in the future for aligned series, we need to remove
// this check here.
long rowCount = getTimeStatistics().getCount();
- for (Statistics<? extends Serializable> statistics : getValueStatisticsList()) {
- if (statistics == null || statistics.hasNullValue(rowCount)) {
+ for (Statistics<? extends Serializable> vStatistics : getValueStatisticsList()) {
+ if (vStatistics == null || vStatistics.hasNullValue(rowCount)) {
return true;
}
}
@@ -118,9 +117,12 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader {
if (paginationController.hasCurOffset(rowCount)) {
paginationController.consumeOffset(rowCount);
return false;
+ } else {
+ return true;
}
+ } else {
+ return valueFilter.satisfy(statistics);
}
- return true;
}
@Override
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index 8064db9748f..0a93e92f423 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -118,18 +118,16 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
}
private boolean pageSatisfy() {
- if (filter != null) {
- // TODO accept valueStatisticsList to filter
- return filter.satisfy(getStatistics());
- } else {
+ Statistics statistics = getStatistics();
+ if (filter == null || filter.allSatisfy(statistics)) {
// For aligned series, When we only query some measurements under an aligned device, if the
// values of these queried measurements at a timestamp are all null, the timestamp will not be
// selected.
// NOTE: if we change the query semantic in the future for aligned series, we need to remove
// this check here.
long rowCount = getTimeStatistics().getCount();
- for (Statistics statistics : getValueStatisticsList()) {
- if (statistics == null || statistics.hasNullValue(rowCount)) {
+ for (Statistics vStatistics : getValueStatisticsList()) {
+ if (vStatistics == null || vStatistics.hasNullValue(rowCount)) {
return true;
}
}
@@ -138,9 +136,13 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
if (paginationController.hasCurOffset(rowCount)) {
paginationController.consumeOffset(rowCount);
return false;
+ } else {
+ return true;
}
+ } else {
+ // TODO accept valueStatisticsList to filter
+ return filter.satisfy(statistics);
}
- return true;
}
@Override