You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2025/04/09 09:08:36 UTC

(pinot) branch master updated: STP-3739: Segment files batch deletion. (#15473)

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new fc8c770fcd STP-3739: Segment files batch deletion. (#15473)
fc8c770fcd is described below

commit fc8c770fcd809b3b37842631a8e73b43c6ea9a9c
Author: Abhishek Bafna <ab...@startree.ai>
AuthorDate: Wed Apr 9 14:38:29 2025 +0530

    STP-3739: Segment files batch deletion. (#15473)
    
    * STP-3739: Segment files batch deletion.
    
    * Add unit tests for the S3 FS.
    
    * GCS batch deletion implementation.
    
    * Handle different bucket batch deletion.
    
    ---------
    
    Co-authored-by: abhishekbafna <ab...@startree.ai>
---
 .../helix/core/PinotHelixResourceManager.java      |   3 +-
 .../helix/core/SegmentDeletionManager.java         | 126 +++++++++++++++------
 .../core/util/SegmentDeletionManagerTest.java      |  34 ++++++
 .../apache/pinot/plugin/filesystem/GcsPinotFS.java |  27 ++++-
 .../apache/pinot/plugin/filesystem/S3PinotFS.java  |  79 +++++++++++++
 .../pinot/plugin/filesystem/S3PinotFSTest.java     |  49 ++++++--
 .../apache/pinot/spi/filesystem/BasePinotFS.java   |  11 ++
 .../pinot/spi/filesystem/NoClosePinotFS.java       |   6 +
 .../org/apache/pinot/spi/filesystem/PinotFS.java   |  10 ++
 9 files changed, 300 insertions(+), 45 deletions(-)

diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 1024adc10f..5fd28e9765 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -2119,7 +2119,8 @@ public class PinotHelixResourceManager {
 
     // Remove all stored segments for the table
     Long retentionPeriodMs = retentionPeriod != null ? TimeUtils.convertPeriodToMillis(retentionPeriod) : null;
-    _segmentDeletionManager.removeSegmentsFromStore(tableNameWithType, getSegmentsFromPropertyStore(tableNameWithType),
+    _segmentDeletionManager.removeSegmentsFromStoreInBatch(tableNameWithType,
+        getSegmentsFromPropertyStore(tableNameWithType),
         retentionPeriodMs);
     LOGGER.info("Deleting table {}: Removed stored segments", tableNameWithType);
 
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index c001042f9a..217f93a84d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -215,6 +215,55 @@ public class SegmentDeletionManager {
     }
   }
 
+  public void removeSegmentsFromStoreInBatch(String tableNameWithType, List<String> segments,
+      @Nullable Long deletedSegmentsRetentionMs) {
+    if (_dataDir == null) {
+      LOGGER.info("dataDir is not configured, won't delete segment from disk for table: {}", tableNameWithType);
+      return;
+    }
+
+    long retentionMs =
+        (deletedSegmentsRetentionMs == null) ? _defaultDeletedSegmentsRetentionMs : deletedSegmentsRetentionMs;
+    String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
+
+    List<URI> filesToDelete = new ArrayList<>();
+    List<URI> metadataFilesToDelete = new ArrayList<>();
+
+    PinotFS pinotFS = PinotFSFactory.create(URIUtils.getUri(_dataDir).getScheme());
+
+    for (String segmentId : segments) {
+      URI fileToDeleteURI = getFileToDeleteURI(rawTableName, segmentId);
+      if (fileToDeleteURI == null) {
+        continue;
+      }
+      try {
+        URI segmentMetadataUri = SegmentPushUtils.generateSegmentMetadataURI(fileToDeleteURI.toString(), segmentId);
+        metadataFilesToDelete.add(segmentMetadataUri);
+      } catch (URISyntaxException e) {
+        LOGGER.warn("Could not generate segment metadata URI for segment: {}", segmentId, e);
+      }
+
+      if (retentionMs <= 0) {
+        filesToDelete.add(fileToDeleteURI);
+      } else {
+        moveSegmentsToDeletedDir(segmentId, deletedSegmentsRetentionMs, rawTableName, pinotFS, fileToDeleteURI);
+      }
+    }
+
+    try {
+      if (!filesToDelete.isEmpty()) {
+        LOGGER.info("Deleting {} segment files", filesToDelete.size());
+        pinotFS.deleteBatch(filesToDelete, true);
+      }
+      if (!metadataFilesToDelete.isEmpty()) {
+        LOGGER.info("Deleting {} segment metadata files", metadataFilesToDelete.size());
+        pinotFS.deleteBatch(metadataFilesToDelete, true);
+      }
+    } catch (IOException e) {
+      LOGGER.warn("Could not delete segment/metadata files", e);
+    }
+  }
+
   private void deleteSegmentMetadataFromStore(PinotFS pinotFS, URI segmentFileUri, String segmentId) {
     // Check if segment metadata exists in remote store and delete it.
     // URI is generated from segment's location and segment name
@@ -241,7 +290,6 @@ public class SegmentDeletionManager {
       String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
       URI fileToDeleteURI = getFileToDeleteURI(rawTableName, segmentId);
       if (fileToDeleteURI == null) {
-        LOGGER.warn("No segment file found for segment: {} in deep store, skipping deletion", segmentId);
         return;
       }
       PinotFS pinotFS = PinotFSFactory.create(fileToDeleteURI.getScheme());
@@ -250,46 +298,56 @@ public class SegmentDeletionManager {
       deleteSegmentMetadataFromStore(pinotFS, fileToDeleteURI, segmentId);
       if (retentionMs <= 0) {
         // delete the segment file instantly if retention is set to zero
-        try {
-          if (pinotFS.delete(fileToDeleteURI, true)) {
-            LOGGER.info("Deleted segment {} from {}", segmentId, fileToDeleteURI.toString());
-          } else {
-            LOGGER.warn("Failed to delete segment {} from {}", segmentId, fileToDeleteURI.toString());
-          }
-        } catch (IOException e) {
-          LOGGER.warn("Could not delete segment {} from {}", segmentId, fileToDeleteURI.toString(), e);
-        }
+        segmentDeletion(segmentId, pinotFS, fileToDeleteURI);
       } else {
-        // move the segment file to deleted segments first and let retention manager handler the deletion
-        String deletedFileName = deletedSegmentsRetentionMs == null ? URIUtils.encode(segmentId)
-            : getDeletedSegmentFileName(URIUtils.encode(segmentId), deletedSegmentsRetentionMs);
-        URI deletedSegmentMoveDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS, rawTableName, deletedFileName);
-        try {
-          if (pinotFS.exists(fileToDeleteURI)) {
-            // Overwrites the file if it already exists in the target directory.
-            if (pinotFS.move(fileToDeleteURI, deletedSegmentMoveDestURI, true)) {
-              // Updates last modified.
-              // Touch is needed here so that removeAgedDeletedSegments() works correctly.
-              pinotFS.touch(deletedSegmentMoveDestURI);
-              LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
-                  deletedSegmentMoveDestURI.toString());
-            } else {
-              LOGGER.warn("Failed to move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
-                  deletedSegmentMoveDestURI.toString());
-            }
-          } else {
-            LOGGER.warn("Failed to find local segment file for segment {}", fileToDeleteURI.toString());
-          }
-        } catch (IOException e) {
-          LOGGER.warn("Could not move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
-              deletedSegmentMoveDestURI.toString(), e);
-        }
+        moveSegmentsToDeletedDir(segmentId, deletedSegmentsRetentionMs, rawTableName, pinotFS, fileToDeleteURI);
       }
     } else {
       LOGGER.info("dataDir is not configured, won't delete segment {} from disk", segmentId);
     }
   }
 
+  private void moveSegmentsToDeletedDir(String segmentId, Long deletedSegmentsRetentionMs, String rawTableName,
+      PinotFS pinotFS,
+      URI fileToDeleteURI) {
+    // move the segment file to deleted segments first and let retention manager handler the deletion
+    String deletedFileName = deletedSegmentsRetentionMs == null ? URIUtils.encode(segmentId)
+        : getDeletedSegmentFileName(URIUtils.encode(segmentId), deletedSegmentsRetentionMs);
+    URI deletedSegmentMoveDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS, rawTableName, deletedFileName);
+    try {
+      if (pinotFS.exists(fileToDeleteURI)) {
+        // Overwrites the file if it already exists in the target directory.
+        if (pinotFS.move(fileToDeleteURI, deletedSegmentMoveDestURI, true)) {
+          // Updates last modified.
+          // Touch is needed here so that removeAgedDeletedSegments() works correctly.
+          pinotFS.touch(deletedSegmentMoveDestURI);
+          LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+              deletedSegmentMoveDestURI.toString());
+        } else {
+          LOGGER.warn("Failed to move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+              deletedSegmentMoveDestURI.toString());
+        }
+      } else {
+        LOGGER.warn("Failed to find local segment file for segment {}", fileToDeleteURI.toString());
+      }
+    } catch (IOException e) {
+      LOGGER.warn("Could not move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+          deletedSegmentMoveDestURI.toString(), e);
+    }
+  }
+
+  private static void segmentDeletion(String segmentId, PinotFS pinotFS, URI fileToDeleteURI) {
+    try {
+      if (pinotFS.delete(fileToDeleteURI, true)) {
+        LOGGER.info("Deleted segment {} from {}", segmentId, fileToDeleteURI.toString());
+      } else {
+        LOGGER.warn("Failed to delete segment {} from {}", segmentId, fileToDeleteURI.toString());
+      }
+    } catch (IOException e) {
+      LOGGER.warn("Could not delete segment {} from {}", segmentId, fileToDeleteURI.toString(), e);
+    }
+  }
+
   /**
    * Retrieves the URI for segment deletion by checking two possible segment file variants in deep store.
    * Looks for the segment file in two formats:
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
index df12b76280..c810151c5c 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
@@ -469,6 +469,40 @@ public class SegmentDeletionManagerTest {
     }, 2000L, 10_000L, "Unable to verify table deletion with retention");
   }
 
+  @Test
+  public void testRemoveSegmentsFromStoreInBatch()
+      throws Exception {
+    Map<String, Object> properties = new HashMap<>();
+    properties.put(CommonConstants.Controller.PREFIX_OF_CONFIG_OF_PINOT_FS_FACTORY + ".class",
+        LocalPinotFS.class.getName());
+    PinotFSFactory.init(new PinotConfiguration(properties));
+
+    HelixAdmin helixAdmin = makeHelixAdmin();
+    ZkHelixPropertyStore<ZNRecord> propertyStore = makePropertyStore();
+    File tempDir = Files.createTempDir();
+    tempDir.deleteOnExit();
+    SegmentDeletionManager deletionManager = new SegmentDeletionManager(
+        tempDir.getAbsolutePath(), helixAdmin, CLUSTER_NAME, propertyStore, 7);
+
+    // create table segment files.
+    List<String> segmentsThatShouldBeDeleted = segmentsThatShouldBeDeleted();
+    createTableAndSegmentFiles(tempDir, segmentsThatShouldBeDeleted);
+    final File tableDir = new File(tempDir.getAbsolutePath() + File.separator + TABLE_NAME);
+    final File deletedTableDir = new File(tempDir.getAbsolutePath() + File.separator + "Deleted_Segments"
+        + File.separator + TABLE_NAME);
+
+    deletionManager.removeSegmentsFromStoreInBatch(TABLE_NAME, segmentsThatShouldBeDeleted, 0L);
+
+    TestUtils.waitForCondition(aVoid -> {
+      try {
+        Assert.assertEquals(tableDir.listFiles().length, 0);
+        Assert.assertTrue(!deletedTableDir.exists() || deletedTableDir.listFiles().length == 0);
+        return true;
+      } catch (Throwable t) {
+        return false;
+      }
+    }, 2000L, 10_000L, "Unable to verify table deletion with retention");
+  }
 
   public void createTableAndSegmentFiles(File tempDir, List<String> segmentIds)
       throws Exception {
diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
index 3f6656ecaa..c58d9528e0 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
@@ -44,6 +44,7 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.Iterator;
 import java.util.List;
 import java.util.function.Consumer;
 import org.apache.commons.io.FileUtils;
@@ -64,7 +65,7 @@ public class GcsPinotFS extends BasePinotFS {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(GcsPinotFS.class);
   // See https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
-  private static final int BATCH_LIMIT = 100;
+  private static final int DELETE_BATCH_LIMIT = 100;
   private Storage _storage;
 
   @Override
@@ -134,6 +135,28 @@ public class GcsPinotFS extends BasePinotFS {
     return delete(new GcsUri(segmentUri), forceDelete);
   }
 
+  @Override
+  public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+      throws IOException {
+    boolean result = true;
+    List<BlobId> blobIds = new ArrayList<>();
+    Iterator<URI> iterator = segmentUris.iterator();
+    while (iterator.hasNext()) {
+      GcsUri gcsUri = new GcsUri(iterator.next());
+      if (existsDirectoryOrBucket(gcsUri)) {
+        result &= delete(gcsUri, forceDelete);
+      } else {
+        blobIds.add(getBlob(gcsUri).getBlobId());
+        if (blobIds.size() >= DELETE_BATCH_LIMIT || !iterator.hasNext()) {
+          List<Boolean> deleted = _storage.delete(blobIds);
+          result &= deleted.stream().allMatch(Boolean::booleanValue);
+          blobIds.clear();
+        }
+      }
+    }
+    return result;
+  }
+
   @Override
   public boolean doMove(URI srcUri, URI dstUri)
       throws IOException {
@@ -428,7 +451,7 @@ public class GcsPinotFS extends BasePinotFS {
     for (Blob blob : page.iterateAll()) {
       results.add(batch.delete(blob.getBlobId()));
       batchSize++;
-      if (batchSize >= BATCH_LIMIT) {
+      if (batchSize >= DELETE_BATCH_LIMIT) {
         batch.submit();
         deleteSucceeded &= results.stream().allMatch(r -> r != null && r.get());
         results = new ArrayList<>();
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
index 0339a973e0..5aa240492e 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
@@ -66,8 +66,11 @@ import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
 import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.Delete;
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
@@ -76,6 +79,7 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
 import software.amazon.awssdk.services.s3.model.MetadataDirective;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -99,6 +103,7 @@ public class S3PinotFS extends BasePinotFS {
   public static final String S3_SCHEME = "s3";
   public static final String S3A_SCHEME = "s3a";
   public static final String SCHEME_SEPARATOR = "://";
+  public static final int DELETE_BATCH_SIZE = 1000;
 
   private S3Client _s3Client;
   private boolean _disableAcl;
@@ -390,6 +395,80 @@ public class S3PinotFS extends BasePinotFS {
     }
   }
 
+  @Override
+  public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+      throws IOException {
+    boolean deletionResult = true;
+    String prevBucket = null;
+    try {
+      List<ObjectIdentifier> objectsToDelete = new ArrayList<>();
+      LOGGER.info("Deleting URIs {} force {}", segmentUris, forceDelete);
+      // initialize the first bucket
+      if (!segmentUris.isEmpty()) {
+        prevBucket = segmentUris.get(0).getHost();
+      }
+      // Iterate through the URIs and delete them
+      for (URI segmentUri : segmentUris) {
+
+        if (isDirectory(segmentUri)) {
+          if (!forceDelete) {
+            Preconditions.checkState(isEmptyDirectory(segmentUri),
+                "ForceDelete flag is not set and directory '%s' is not empty", segmentUri);
+          }
+
+          // Recursively list files in the directory
+          LOGGER.info("Recursively deleting files in directory {}", segmentUri);
+          List<URI> filesInDir = listFiles(segmentUri);
+          deletionResult &= deleteBatch(filesInDir, forceDelete);
+        } else {
+          String key = sanitizePath(segmentUri.getPath());
+          objectsToDelete.add(ObjectIdentifier.builder().key(key).build());
+          String bucket = segmentUri.getHost();
+
+          // If batch reaches max size, process the batch
+          if (objectsToDelete.size() >= DELETE_BATCH_SIZE || !bucket.equals(prevBucket)) {
+            deletionResult &= processBatch(prevBucket, objectsToDelete);
+            objectsToDelete.clear();  // Clear list for the next batch
+            prevBucket = bucket;
+          }
+        }
+      }
+
+      // Process remaining files in the last batch
+      if (!objectsToDelete.isEmpty()) {
+        deletionResult &= processBatch(prevBucket, objectsToDelete);
+      }
+      return deletionResult;
+    } catch (S3Exception e) {
+      throw e;
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+  }
+
+  private boolean processBatch(String bucket, List<ObjectIdentifier> objectsToDelete) {
+    LOGGER.info("Deleting batch of {} objects", objectsToDelete.size());
+    DeleteObjectsRequest deleteRequest = DeleteObjectsRequest.builder()
+        .bucket(bucket)
+        .delete(Delete.builder().objects(objectsToDelete).build())
+        .build();
+
+    DeleteObjectsResponse deleteResponse = _s3Client.deleteObjects(deleteRequest);
+    LOGGER.info("Failed to delete {} objects", deleteResponse.hasErrors() ? deleteResponse.errors().size() : 0);
+    return deleteResponse.deleted().size() == objectsToDelete.size();
+  }
+
+  private List<URI> listFiles(URI directoryUri)
+      throws IOException {
+    String[] listedFiles = listFiles(directoryUri, true);
+    List<URI> fileUris = new ArrayList<>();
+    for (String filePath : listedFiles) {
+      fileUris.add(URI.create(filePath));
+    }
+    LOGGER.info("Files in directory {}: {}", directoryUri, fileUris);
+    return fileUris;
+  }
+
   @Override
   public boolean delete(URI segmentUri, boolean forceDelete)
       throws IOException {
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
index 8c9ed7f5fe..53bbb96dd4 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
@@ -72,7 +72,7 @@ public class S3PinotFSTest {
 
   @DataProvider(name = "scheme")
   public static Object[][] schemes() {
-    return new Object[][] { { S3_SCHEME }, { S3A_SCHEME } };
+    return new Object[][]{{S3_SCHEME}, {S3A_SCHEME}};
   }
 
   @BeforeClass
@@ -502,15 +502,48 @@ public class S3PinotFSTest {
     }
   }
 
+  public void testDeleteBatch()
+      throws IOException {
+    String[] originalFiles = new String[]{"a-delete-batch.txt", "b-delete-batch.txt", "c-delete-batch.txt"};
+    String folderName = "my-files-batch";
+
+    for (String fileName : originalFiles) {
+      createEmptyFile(folderName, fileName);
+    }
+    // Create a sub folder to delete
+    String subFolderName = folderName + DELIMITER + "subfolder";
+    for (String fileName : new String[] {"subfolder-a-delete-batch.txt", "subfolder-b-delete-batch.txt"}) {
+      createEmptyFile(subFolderName, fileName);
+    }
+
+    // Create a list of URIs to delete
+    List<URI> filesToDelete = Arrays.stream(originalFiles)
+        .map(fileName -> URI.create(String.format(FILE_FORMAT, S3_SCHEME, BUCKET, folderName + DELIMITER + fileName)))
+        .collect(Collectors.toList());
+    filesToDelete.add(URI.create(String.format(FILE_FORMAT, S3_SCHEME, BUCKET, subFolderName)));
+
+    boolean deleteResult = _s3PinotFS.deleteBatch(filesToDelete, true);
+
+    Assert.assertTrue(deleteResult);
+
+    ListObjectsV2Response listObjectsV2Response =
+        _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));
+    String[] actualResponse =
+        listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("delete-batch"))
+            .toArray(String[]::new);
+
+    Assert.assertEquals(actualResponse.length, 0);
+  }
+
   @DataProvider(name = "storageClasses")
   public Object[][] createStorageClasses() {
-    return new Object[][] {
-      { null, S3_SCHEME },
-      { StorageClass.STANDARD, S3_SCHEME },
-      { StorageClass.INTELLIGENT_TIERING, S3_SCHEME },
-      { null, S3A_SCHEME },
-      { StorageClass.STANDARD, S3A_SCHEME },
-      { StorageClass.INTELLIGENT_TIERING, S3A_SCHEME },
+    return new Object[][]{
+        {null, S3_SCHEME},
+        {StorageClass.STANDARD, S3_SCHEME},
+        {StorageClass.INTELLIGENT_TIERING, S3_SCHEME},
+        {null, S3A_SCHEME},
+        {StorageClass.STANDARD, S3A_SCHEME},
+        {StorageClass.INTELLIGENT_TIERING, S3A_SCHEME},
     };
   }
 
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
index bd76bdf740..8d2724f25a 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
@@ -23,6 +23,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,6 +31,16 @@ import org.slf4j.LoggerFactory;
 public abstract class BasePinotFS implements PinotFS {
   private static final Logger LOGGER = LoggerFactory.getLogger(BasePinotFS.class);
 
+  @Override
+  public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+      throws IOException {
+    boolean result = true;
+    for (URI segmentUri : segmentUris) {
+      result &= delete(segmentUri, forceDelete);
+    }
+    return result;
+  }
+
   @Override
   public boolean move(URI srcUri, URI dstUri, boolean overwrite)
       throws IOException {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
index d9fc002c7e..de29fe4cb8 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
@@ -50,6 +50,12 @@ public class NoClosePinotFS implements PinotFS {
     return _delegate.delete(segmentUri, forceDelete);
   }
 
+  @Override
+  public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+      throws IOException {
+    return _delegate.deleteBatch(segmentUris, forceDelete);
+  }
+
   @Override
   public boolean move(URI srcUri, URI dstUri, boolean overwrite)
       throws IOException {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
index 286a02c7d4..863df527f9 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
@@ -70,6 +70,16 @@ public interface PinotFS extends Closeable, Serializable {
   boolean delete(URI segmentUri, boolean forceDelete)
       throws IOException;
 
+  /**
+   * Deletes the files at the locations provided. If the segmentUri is a directory, it will delete the entire directory.
+   * @param segmentUris List of URIs of the segments
+   * @param forceDelete true if we want the uri and any sub-uris to always be deleted, false if we want delete to fail
+   * @return true if delete is successful else false
+   * @throws IOException on IO failure, e.g Uri is not present or not valid
+   */
+  boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+      throws IOException;
+
   /**
    * Moves the file or directory from the src to dst. Does not keep the original file. If the dst has parent directories
    * that haven't been created, this method will create all the necessary parent directories.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org