You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2020/02/21 02:10:34 UTC
[incubator-iotdb] 01/01: add TsFileFilter
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch add_TsFileFilter
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 9355a1fa650e0dedfb4425be093d6cba6e9e102d
Author: jt2594838 <jt...@163.com>
AuthorDate: Fri Feb 21 10:10:18 2020 +0800
add TsFileFilter
---
.gitignore | 3 +++
.../iotdb/db/query/executor/EngineExecutor.java | 4 +--
.../java/org/apache/iotdb/db/query/fill/IFill.java | 3 ++-
.../apache/iotdb/db/query/filter/TsFileFilter.java | 31 ++++++++++++++++++++++
.../seriesRelated/SeriesReaderWithValueFilter.java | 6 +++--
.../SeriesReaderWithoutValueFilter.java | 6 ++++-
.../query/timegenerator/EngineNodeConstructor.java | 3 ++-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 14 ++++++++++
8 files changed, 63 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
index 70913bb..34f0e12 100644
--- a/.gitignore
+++ b/.gitignore
@@ -80,3 +80,6 @@ tsfile/src/test/resources/*.ts
local-snapshots-dir/
venv/
+partitions.tmp
+partitions
+node_identifier
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
index 0a10375..456eeb3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
@@ -78,7 +78,7 @@ public class EngineExecutor {
TSDataType dataType = deduplicatedDataTypes.get(i);
ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context,
- true);
+ true, null);
readersOfSelectedSeries.add(reader);
}
@@ -104,7 +104,7 @@ public class EngineExecutor {
TSDataType dataType = deduplicatedDataTypes.get(i);
ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context,
- true);
+ true, null);
readersOfSelectedSeries.add(reader);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
index 5db57eb..4fb17ed 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
@@ -54,7 +54,8 @@ public abstract class IFill {
void constructReaders(Path path, QueryContext context, long beforeRange)
throws IOException, StorageEngineException {
Filter timeFilter = constructFilter(beforeRange);
- allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true);
+ allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true,
+ null);
}
public abstract IPointReader getFillResult() throws IOException, UnSupportedFillTypeException;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java
new file mode 100644
index 0000000..97711ef
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.filter;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+/**
+ * TsFileFilter is used to filter unwanted TsFiles in a QueryDataSource, to better support data
+ * partitioning in the distributed version and other features.
+ */
+@FunctionalInterface
+public interface TsFileFilter {
+ boolean fileNotSatisfy(TsFileResource resource);
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
index 83f0d3c..dda88cd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.seriesRelated;
import java.io.IOException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.reader.IBatchReader;
@@ -48,9 +49,10 @@ public class SeriesReaderWithValueFilter extends SeriesReaderWithoutValueFilter
private TimeValuePair timeValuePair;
private BatchData batchData;
- public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter, QueryContext context)
+ public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter,
+ QueryContext context, TsFileFilter fileFilter)
throws StorageEngineException, IOException {
- super(seriesPath, dataType, filter, context, false);
+ super(seriesPath, dataType, filter, context, false, fileFilter);
this.filter = filter;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
index 27bd19c..0ab2692 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
@@ -23,9 +23,11 @@ import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.ManagedSeriesReader;
import org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader;
import org.apache.iotdb.db.query.reader.resourceRelated.NewUnseqResourceMergeReader;
+import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.BatchData;
@@ -83,9 +85,11 @@ public class SeriesReaderWithoutValueFilter implements ManagedSeriesReader {
* to. We do not push down value filter to unsequence readers
*/
public SeriesReaderWithoutValueFilter(Path seriesPath, TSDataType dataType, Filter timeFilter,
- QueryContext context, boolean pushdownUnseq) throws StorageEngineException, IOException {
+ QueryContext context, boolean pushdownUnseq, TsFileFilter fileFilter) throws StorageEngineException,
+ IOException {
QueryDataSource queryDataSource = QueryResourceManager.getInstance()
.getQueryDataSource(seriesPath, context);
+ QueryUtils.filterQueryDataSource(queryDataSource, fileFilter);
timeFilter = queryDataSource.updateTimeFilter(timeFilter);
// reader for sequence resources
diff --git a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
index 41fc515..0a9891d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
@@ -56,7 +56,8 @@ public class EngineNodeConstructor extends AbstractNodeConstructor {
Filter filter = ((SingleSeriesExpression) expression).getFilter();
Path path = ((SingleSeriesExpression) expression).getSeriesPath();
TSDataType dataType = MManager.getInstance().getSeriesType(path.getFullPath());
- return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context));
+ return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context
+ , null));
} catch (IOException | PathException e) {
throw new StorageEngineException(e.getMessage());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 03e90e3..e681832 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -22,6 +22,9 @@ package org.apache.iotdb.db.utils;
import java.util.List;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
public class QueryUtils {
@@ -73,4 +76,15 @@ public class QueryUtils {
}
return false;
}
+
+ // remove files that do not satisfy the filter
+ public static void filterQueryDataSource(QueryDataSource queryDataSource, TsFileFilter fileFilter) {
+ if (fileFilter == null) {
+ return;
+ }
+ List<TsFileResource> seqResources = queryDataSource.getSeqResources();
+ List<TsFileResource> unseqResources = queryDataSource.getUnseqResources();
+ seqResources.removeIf(fileFilter::fileNotSatisfy);
+ unseqResources.removeIf(fileFilter::fileNotSatisfy);
+ }
}