(pinot) branch master updated: Fixing the logic to only process deep store untracked segments in the offline table part for a hybrid table (#16107)

[xi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 4af0c8f153 Fixing the logic to only process deep store untracked segments in the offline table part for a hybrid table (#16107)
4af0c8f153 is described below

commit 4af0c8f15367f2e0371d1cf2daa2fc4b9fd13a0a
Author: Xiang Fu <xi...@gmail.com>
AuthorDate: Sun Jun 15 03:21:55 2025 +0800

    Fixing the logic to only process deep store untracked segments in the offline table part for a hybrid table (#16107)
---
 .../helix/core/retention/RetentionManager.java     | 29 +++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
index e139a93cc6..984bfbd527 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
@@ -301,6 +301,18 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
   private List<String> getSegmentsToDeleteFromDeepstore(String tableNameWithType, RetentionStrategy retentionStrategy,
       List<SegmentZKMetadata> segmentZKMetadataList, int untrackedSegmentsDeletionBatchSize) {
     List<String> segmentsToDelete = new ArrayList<>();
+    String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
+    boolean isHybridTable = _pinotHelixResourceManager.hasOfflineTable(rawTableName)
+        && _pinotHelixResourceManager.hasRealtimeTable(rawTableName);
+    if (isHybridTable && TableNameBuilder.isRealtimeTableResource(tableNameWithType)) {
+      // If it is a hybrid table, we don't need to scan deep store for untracked segments when processing the
+      // realtime table.
+      // This is because realtime tables are expected to have short retention periods, so scanning deep store for
+      // untracked segments is not necessary.
+      LOGGER.info("Skipping deep store scan for untracked segments for realtime table: {} as it's a hybrid table",
+          tableNameWithType);
+      return segmentsToDelete;
+    }
 
     if (!_untrackedSegmentDeletionEnabled) {
       LOGGER.info(
@@ -317,8 +329,20 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
       return segmentsToDelete;
     }
 
-    List<String> segmentsPresentInZK =
-        segmentZKMetadataList.stream().map(SegmentZKMetadata::getSegmentName).collect(Collectors.toList());
+    List<String> segmentsPresentInZK;
+    if (isHybridTable) {
+      segmentsPresentInZK = new ArrayList<>();
+      // This must be the OFFLINE table
+      segmentsPresentInZK.addAll(
+          segmentZKMetadataList.stream().map(SegmentZKMetadata::getSegmentName).collect(Collectors.toList()));
+      // Add segments from the REALTIME table as well
+      segmentsPresentInZK.addAll(
+          _pinotHelixResourceManager.getSegmentsFor(TableNameBuilder.REALTIME.tableNameWithType(rawTableName), false));
+    } else {
+      segmentsPresentInZK =
+          segmentZKMetadataList.stream().map(SegmentZKMetadata::getSegmentName).collect(Collectors.toList());
+    }
+
     try {
       LOGGER.info("Fetch segments present in deep store that are beyond retention period for table: {}",
           tableNameWithType);
@@ -340,7 +364,6 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
     return segmentsToDelete;
   }
 
-
   /**
    * Identifies segments in deepstore that are ready for deletion based on the retention strategy.
    *


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org