You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/22 12:12:47 UTC

(iotdb) 01/01: Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException (#13869)

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch cp-13869
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 58965c53f970600dde100b9a1445796177f00203
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue Oct 22 20:00:55 2024 +0800

    Pipe: Remove already deleted tsfile resource from historical extraction list to prevent FileNotFoundException (#13869)
    
    (cherry picked from commit ce53f9c160e49cebf747151bd6e7b65c53185953)
---
 .../PipeHistoricalDataRegionTsFileExtractor.java   | 34 +++++++++++++---------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index 917bd6ae594..c0e0561a45c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -435,13 +435,16 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
             tsFileManager.getTsFileList(true).stream()
                 .filter(
                     resource ->
-                        // Some resource may not be closed due to the control of
-                        // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
-                        !resource.isClosed()
-                            || mayTsFileContainUnprocessedData(resource)
-                                && isTsFileResourceOverlappedWithTimeRange(resource)
-                                && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
-                                && mayTsFileResourceOverlappedWithPattern(resource))
+                        // Some resource is marked as deleted but not removed from the list.
+                        !resource.isDeleted()
+                            && (
+                            // Some resource may not be closed due to the control of
+                            // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+                            !resource.isClosed()
+                                || mayTsFileContainUnprocessedData(resource)
+                                    && isTsFileResourceOverlappedWithTimeRange(resource)
+                                    && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
+                                    && mayTsFileResourceOverlappedWithPattern(resource)))
                 .collect(Collectors.toList());
         resourceList.addAll(sequenceTsFileResources);
 
@@ -449,13 +452,16 @@ public class PipeHistoricalDataRegionTsFileExtractor implements PipeHistoricalDa
             tsFileManager.getTsFileList(false).stream()
                 .filter(
                     resource ->
-                        // Some resource may not be closed due to the control of
-                        // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
-                        !resource.isClosed()
-                            || mayTsFileContainUnprocessedData(resource)
-                                && isTsFileResourceOverlappedWithTimeRange(resource)
-                                && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
-                                && mayTsFileResourceOverlappedWithPattern(resource))
+                        // Some resource is marked as deleted but not removed from the list.
+                        !resource.isDeleted()
+                            && (
+                            // Some resource may not be closed due to the control of
+                            // PIPE_MIN_FLUSH_INTERVAL_IN_MS. We simply ignore them.
+                            !resource.isClosed()
+                                || mayTsFileContainUnprocessedData(resource)
+                                    && isTsFileResourceOverlappedWithTimeRange(resource)
+                                    && isTsFileGeneratedAfterExtractionTimeLowerBound(resource)
+                                    && mayTsFileResourceOverlappedWithPattern(resource)))
                 .collect(Collectors.toList());
         resourceList.addAll(unsequenceTsFileResources);