You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/11 09:00:25 UTC
(iotdb) branch table-pattern-PipeHistoricalDataRegionTsFileExtractor created (now f3a7dc8ac3b)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch table-pattern-PipeHistoricalDataRegionTsFileExtractor
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at f3a7dc8ac3b Pipe: Apply table pattern matching in historical extractor
This branch includes the following new commits:
new f3a7dc8ac3b Pipe: Apply table pattern matching in historical extractor
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Pipe: Apply table pattern matching in historical extractor
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch table-pattern-PipeHistoricalDataRegionTsFileExtractor
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f3a7dc8ac3b03e68576486cd5115da72a26bb0c4
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri Oct 11 17:00:13 2024 +0800
Pipe: Apply table pattern matching in historical extractor
---
.../PipeHistoricalDataRegionTsFileExtractor.java | 23 +++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index 332751e22dc..dc019106834 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -48,6 +48,7 @@ import org.apache.iotdb.pipe.api.event.Event;
import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,6 +86,8 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstan
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_START_TIME_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODS_ENABLE_KEY;
import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_START_TIME_KEY;
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT;
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDataRegionExtractor {
@@ -103,7 +106,7 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
private int dataRegionId;
private TreePattern treePattern;
- private TablePattern tablePattern; // TODO: use like treePattern
+ private TablePattern tablePattern;
private boolean isDbNameCoveredByPattern = false;
private boolean isHistoricalExtractorEnabled = false;
@@ -545,8 +548,22 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
return true;
}
- // TODO: consider tablePattern
- return deviceSet.stream().anyMatch(deviceID -> treePattern.mayOverlapWithDevice(deviceID));
+ return deviceSet.stream()
+ .anyMatch(
+ deviceID -> {
+ if (deviceID instanceof PlainDeviceID
+ || deviceID.getTableName().startsWith(PATH_ROOT + PATH_SEPARATOR)
+ || deviceID.getTableName().equals(PATH_ROOT)) {
+ // In case of tree model deviceID
+ return treePattern.isTreeModelDataAllowedToBeCaptured()
+ && treePattern.mayOverlapWithDevice(deviceID);
+ } else {
+ // In case of table model deviceID
+ return tablePattern.isTableModelDataAllowedToBeCaptured()
+ && tablePattern.matchesDatabase(resource.getDatabaseName())
+ && tablePattern.matchesTable(deviceID.getTableName());
+ }
+ });
}
private boolean isTsFileResourceOverlappedWithTimeRange(final TsFileResource resource) {