You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2020/02/21 02:10:33 UTC
[incubator-iotdb] branch add_TsFileFilter created (now 9355a1f)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch add_TsFileFilter
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 9355a1f add TsFileFilter
This branch includes the following new commits:
new 9355a1f add TsFileFilter
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add TsFileFilter
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch add_TsFileFilter
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 9355a1fa650e0dedfb4425be093d6cba6e9e102d
Author: jt2594838 <jt...@163.com>
AuthorDate: Fri Feb 21 10:10:18 2020 +0800
add TsFileFilter
---
.gitignore | 3 +++
.../iotdb/db/query/executor/EngineExecutor.java | 4 +--
.../java/org/apache/iotdb/db/query/fill/IFill.java | 3 ++-
.../apache/iotdb/db/query/filter/TsFileFilter.java | 31 ++++++++++++++++++++++
.../seriesRelated/SeriesReaderWithValueFilter.java | 6 +++--
.../SeriesReaderWithoutValueFilter.java | 6 ++++-
.../query/timegenerator/EngineNodeConstructor.java | 3 ++-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 14 ++++++++++
8 files changed, 63 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
index 70913bb..34f0e12 100644
--- a/.gitignore
+++ b/.gitignore
@@ -80,3 +80,6 @@ tsfile/src/test/resources/*.ts
local-snapshots-dir/
venv/
+partitions.tmp
+partitions
+node_identifier
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
index 0a10375..456eeb3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
@@ -78,7 +78,7 @@ public class EngineExecutor {
TSDataType dataType = deduplicatedDataTypes.get(i);
ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context,
- true);
+ true, null);
readersOfSelectedSeries.add(reader);
}
@@ -104,7 +104,7 @@ public class EngineExecutor {
TSDataType dataType = deduplicatedDataTypes.get(i);
ManagedSeriesReader reader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context,
- true);
+ true, null);
readersOfSelectedSeries.add(reader);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
index 5db57eb..4fb17ed 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
@@ -54,7 +54,8 @@ public abstract class IFill {
void constructReaders(Path path, QueryContext context, long beforeRange)
throws IOException, StorageEngineException {
Filter timeFilter = constructFilter(beforeRange);
- allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true);
+ allDataReader = new SeriesReaderWithoutValueFilter(path, dataType, timeFilter, context, true,
+ null);
}
public abstract IPointReader getFillResult() throws IOException, UnSupportedFillTypeException;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java
new file mode 100644
index 0000000..97711ef
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/filter/TsFileFilter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.filter;
+
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+/**
+ * TsFileFilter is used to filter unwanted TsFiles in a QueryDataSource, to better support data
+ * partitioning in the distributed version and other features.
+ */
+@FunctionalInterface
+public interface TsFileFilter {
+ boolean fileNotSatisfy(TsFileResource resource);
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
index 83f0d3c..dda88cd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithValueFilter.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.query.reader.seriesRelated;
import java.io.IOException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.reader.IBatchReader;
@@ -48,9 +49,10 @@ public class SeriesReaderWithValueFilter extends SeriesReaderWithoutValueFilter
private TimeValuePair timeValuePair;
private BatchData batchData;
- public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter, QueryContext context)
+ public SeriesReaderWithValueFilter(Path seriesPath, TSDataType dataType, Filter filter,
+ QueryContext context, TsFileFilter fileFilter)
throws StorageEngineException, IOException {
- super(seriesPath, dataType, filter, context, false);
+ super(seriesPath, dataType, filter, context, false, fileFilter);
this.filter = filter;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
index 27bd19c..0ab2692 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/seriesRelated/SeriesReaderWithoutValueFilter.java
@@ -23,9 +23,11 @@ import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.ManagedSeriesReader;
import org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader;
import org.apache.iotdb.db.query.reader.resourceRelated.NewUnseqResourceMergeReader;
+import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.BatchData;
@@ -83,9 +85,11 @@ public class SeriesReaderWithoutValueFilter implements ManagedSeriesReader {
* to. We do not push down value filter to unsequence readers
*/
public SeriesReaderWithoutValueFilter(Path seriesPath, TSDataType dataType, Filter timeFilter,
- QueryContext context, boolean pushdownUnseq) throws StorageEngineException, IOException {
+ QueryContext context, boolean pushdownUnseq, TsFileFilter fileFilter) throws StorageEngineException,
+ IOException {
QueryDataSource queryDataSource = QueryResourceManager.getInstance()
.getQueryDataSource(seriesPath, context);
+ QueryUtils.filterQueryDataSource(queryDataSource, fileFilter);
timeFilter = queryDataSource.updateTimeFilter(timeFilter);
// reader for sequence resources
diff --git a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
index 41fc515..0a9891d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
@@ -56,7 +56,8 @@ public class EngineNodeConstructor extends AbstractNodeConstructor {
Filter filter = ((SingleSeriesExpression) expression).getFilter();
Path path = ((SingleSeriesExpression) expression).getSeriesPath();
TSDataType dataType = MManager.getInstance().getSeriesType(path.getFullPath());
- return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context));
+ return new EngineLeafNode(new SeriesReaderWithValueFilter(path, dataType, filter, context
+ , null));
} catch (IOException | PathException e) {
throw new StorageEngineException(e.getMessage());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 03e90e3..e681832 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -22,6 +22,9 @@ package org.apache.iotdb.db.utils;
import java.util.List;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
public class QueryUtils {
@@ -73,4 +76,15 @@ public class QueryUtils {
}
return false;
}
+
+ // remove files that do not satisfy the filter
+ public static void filterQueryDataSource(QueryDataSource queryDataSource, TsFileFilter fileFilter) {
+ if (fileFilter == null) {
+ return;
+ }
+ List<TsFileResource> seqResources = queryDataSource.getSeqResources();
+ List<TsFileResource> unseqResources = queryDataSource.getUnseqResources();
+ seqResources.removeIf(fileFilter::fileNotSatisfy);
+ unseqResources.removeIf(fileFilter::fileNotSatisfy);
+ }
}