You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/22 10:13:46 UTC

(iotdb) branch pipe-filter-deleted-historical-files created (now fd78ff796ea)

This is an automated email from the ASF dual-hosted git repository.

rong pushed a change to branch pipe-filter-deleted-historical-files
in repository https://gitbox.apache.org/repos/asf/iotdb.git


      at fd78ff796ea Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException

This branch includes the following new commits:

     new fd78ff796ea Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(iotdb) 01/01: Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException

Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch pipe-filter-deleted-historical-files
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit fd78ff796eacb643f4034a5bd29ba9eead41b243
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue Oct 22 18:13:37 2024 +0800

    Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException
---
 ...oricalDataRegionTsFileAndDeletionExtractor.java | 34 +++++++++++++---------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
index e4fac1cdabe..23f65311303 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java
@@ -517,13 +517,16 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
           tsFileManager.getTsFileList(true).stream()
               .filter(
                   resource ->
-                      // Some resource may not be closed due to the control of
-                      // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
-                      !resource.isClosed()
-                          || mayTsFileContainUnprocessedData(resource)
-                              && isTsFileResourceOverlappedWithTimeRange(resource)
-                              && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
-                              && mayTsFileResourceOverlappedWithPattern(resource))
+                      // Some resource is marked as deleted but not removed from the list.
+                      !resource.isDeleted()
+                          && (
+                          // Some resource may not be closed due to the control of
+                          // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+                          !resource.isClosed()
+                              || mayTsFileContainUnprocessedData(resource)
+                                  && isTsFileResourceOverlappedWithTimeRange(resource)
+                                  && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
+                                  && mayTsFileResourceOverlappedWithPattern(resource)))
               .collect(Collectors.toList());
       resourceList.addAll(sequenceTsFileResources);
 
@@ -531,13 +534,16 @@ public class PipeHistoricalDataRegionTsFileAndDeletionExtractor
           tsFileManager.getTsFileList(false).stream()
               .filter(
                   resource ->
-                      // Some resource may not be closed due to the control of
-                      // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
-                      !resource.isClosed()
-                          || mayTsFileContainUnprocessedData(resource)
-                              && isTsFileResourceOverlappedWithTimeRange(resource)
-                              && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
-                              && mayTsFileResourceOverlappedWithPattern(resource))
+                      // Some resource is marked as deleted but not removed from the list.
+                      !resource.isDeleted()
+                          && (
+                          // Some resource may not be closed due to the control of
+                          // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+                          !resource.isClosed()
+                              || mayTsFileContainUnprocessedData(resource)
+                                  && isTsFileResourceOverlappedWithTimeRange(resource)
+                                  && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
+                                  && mayTsFileResourceOverlappedWithPattern(resource)))
               .collect(Collectors.toList());
       resourceList.addAll(unsequenceTsFileResources);