You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2025/04/09 09:08:36 UTC
(pinot) branch master updated: STP-3739: Segment files batch deletion. (#15473)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fc8c770fcd STP-3739: Segment files batch deletion. (#15473)
fc8c770fcd is described below
commit fc8c770fcd809b3b37842631a8e73b43c6ea9a9c
Author: Abhishek Bafna <ab...@startree.ai>
AuthorDate: Wed Apr 9 14:38:29 2025 +0530
STP-3739: Segment files batch deletion. (#15473)
* STP-3739: Segment files batch deletion.
* Add unit tests for the S3 FS.
* GCS batch deletion implementation.
* Handle different bucket batch deletion.
---------
Co-authored-by: abhishekbafna <ab...@startree.ai>
---
.../helix/core/PinotHelixResourceManager.java | 3 +-
.../helix/core/SegmentDeletionManager.java | 126 +++++++++++++++------
.../core/util/SegmentDeletionManagerTest.java | 34 ++++++
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 27 ++++-
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 79 +++++++++++++
.../pinot/plugin/filesystem/S3PinotFSTest.java | 49 ++++++--
.../apache/pinot/spi/filesystem/BasePinotFS.java | 11 ++
.../pinot/spi/filesystem/NoClosePinotFS.java | 6 +
.../org/apache/pinot/spi/filesystem/PinotFS.java | 10 ++
9 files changed, 300 insertions(+), 45 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 1024adc10f..5fd28e9765 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -2119,7 +2119,8 @@ public class PinotHelixResourceManager {
// Remove all stored segments for the table
Long retentionPeriodMs = retentionPeriod != null ? TimeUtils.convertPeriodToMillis(retentionPeriod) : null;
- _segmentDeletionManager.removeSegmentsFromStore(tableNameWithType, getSegmentsFromPropertyStore(tableNameWithType),
+ _segmentDeletionManager.removeSegmentsFromStoreInBatch(tableNameWithType,
+ getSegmentsFromPropertyStore(tableNameWithType),
retentionPeriodMs);
LOGGER.info("Deleting table {}: Removed stored segments", tableNameWithType);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index c001042f9a..217f93a84d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -215,6 +215,55 @@ public class SegmentDeletionManager {
}
}
+ public void removeSegmentsFromStoreInBatch(String tableNameWithType, List<String> segments,
+ @Nullable Long deletedSegmentsRetentionMs) {
+ if (_dataDir == null) {
+ LOGGER.info("dataDir is not configured, won't delete segment from disk for table: {}", tableNameWithType);
+ return;
+ }
+
+ long retentionMs =
+ (deletedSegmentsRetentionMs == null) ? _defaultDeletedSegmentsRetentionMs : deletedSegmentsRetentionMs;
+ String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
+
+ List<URI> filesToDelete = new ArrayList<>();
+ List<URI> metadataFilesToDelete = new ArrayList<>();
+
+ PinotFS pinotFS = PinotFSFactory.create(URIUtils.getUri(_dataDir).getScheme());
+
+ for (String segmentId : segments) {
+ URI fileToDeleteURI = getFileToDeleteURI(rawTableName, segmentId);
+ if (fileToDeleteURI == null) {
+ continue;
+ }
+ try {
+ URI segmentMetadataUri = SegmentPushUtils.generateSegmentMetadataURI(fileToDeleteURI.toString(), segmentId);
+ metadataFilesToDelete.add(segmentMetadataUri);
+ } catch (URISyntaxException e) {
+ LOGGER.warn("Could not generate segment metadata URI for segment: {}", segmentId, e);
+ }
+
+ if (retentionMs <= 0) {
+ filesToDelete.add(fileToDeleteURI);
+ } else {
+ moveSegmentsToDeletedDir(segmentId, deletedSegmentsRetentionMs, rawTableName, pinotFS, fileToDeleteURI);
+ }
+ }
+
+ try {
+ if (!filesToDelete.isEmpty()) {
+ LOGGER.info("Deleting {} segment files", filesToDelete.size());
+ pinotFS.deleteBatch(filesToDelete, true);
+ }
+ if (!metadataFilesToDelete.isEmpty()) {
+ LOGGER.info("Deleting {} segment metadata files", metadataFilesToDelete.size());
+ pinotFS.deleteBatch(metadataFilesToDelete, true);
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Could not delete segment/metadata files", e);
+ }
+ }
+
private void deleteSegmentMetadataFromStore(PinotFS pinotFS, URI segmentFileUri, String segmentId) {
// Check if segment metadata exists in remote store and delete it.
// URI is generated from segment's location and segment name
@@ -241,7 +290,6 @@ public class SegmentDeletionManager {
String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
URI fileToDeleteURI = getFileToDeleteURI(rawTableName, segmentId);
if (fileToDeleteURI == null) {
- LOGGER.warn("No segment file found for segment: {} in deep store, skipping deletion", segmentId);
return;
}
PinotFS pinotFS = PinotFSFactory.create(fileToDeleteURI.getScheme());
@@ -250,46 +298,56 @@ public class SegmentDeletionManager {
deleteSegmentMetadataFromStore(pinotFS, fileToDeleteURI, segmentId);
if (retentionMs <= 0) {
// delete the segment file instantly if retention is set to zero
- try {
- if (pinotFS.delete(fileToDeleteURI, true)) {
- LOGGER.info("Deleted segment {} from {}", segmentId, fileToDeleteURI.toString());
- } else {
- LOGGER.warn("Failed to delete segment {} from {}", segmentId, fileToDeleteURI.toString());
- }
- } catch (IOException e) {
- LOGGER.warn("Could not delete segment {} from {}", segmentId, fileToDeleteURI.toString(), e);
- }
+ segmentDeletion(segmentId, pinotFS, fileToDeleteURI);
} else {
- // move the segment file to deleted segments first and let retention manager handler the deletion
- String deletedFileName = deletedSegmentsRetentionMs == null ? URIUtils.encode(segmentId)
- : getDeletedSegmentFileName(URIUtils.encode(segmentId), deletedSegmentsRetentionMs);
- URI deletedSegmentMoveDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS, rawTableName, deletedFileName);
- try {
- if (pinotFS.exists(fileToDeleteURI)) {
- // Overwrites the file if it already exists in the target directory.
- if (pinotFS.move(fileToDeleteURI, deletedSegmentMoveDestURI, true)) {
- // Updates last modified.
- // Touch is needed here so that removeAgedDeletedSegments() works correctly.
- pinotFS.touch(deletedSegmentMoveDestURI);
- LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
- deletedSegmentMoveDestURI.toString());
- } else {
- LOGGER.warn("Failed to move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
- deletedSegmentMoveDestURI.toString());
- }
- } else {
- LOGGER.warn("Failed to find local segment file for segment {}", fileToDeleteURI.toString());
- }
- } catch (IOException e) {
- LOGGER.warn("Could not move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
- deletedSegmentMoveDestURI.toString(), e);
- }
+ moveSegmentsToDeletedDir(segmentId, deletedSegmentsRetentionMs, rawTableName, pinotFS, fileToDeleteURI);
}
} else {
LOGGER.info("dataDir is not configured, won't delete segment {} from disk", segmentId);
}
}
+ private void moveSegmentsToDeletedDir(String segmentId, Long deletedSegmentsRetentionMs, String rawTableName,
+ PinotFS pinotFS,
+ URI fileToDeleteURI) {
+ // move the segment file to deleted segments first and let retention manager handler the deletion
+ String deletedFileName = deletedSegmentsRetentionMs == null ? URIUtils.encode(segmentId)
+ : getDeletedSegmentFileName(URIUtils.encode(segmentId), deletedSegmentsRetentionMs);
+ URI deletedSegmentMoveDestURI = URIUtils.getUri(_dataDir, DELETED_SEGMENTS, rawTableName, deletedFileName);
+ try {
+ if (pinotFS.exists(fileToDeleteURI)) {
+ // Overwrites the file if it already exists in the target directory.
+ if (pinotFS.move(fileToDeleteURI, deletedSegmentMoveDestURI, true)) {
+ // Updates last modified.
+ // Touch is needed here so that removeAgedDeletedSegments() works correctly.
+ pinotFS.touch(deletedSegmentMoveDestURI);
+ LOGGER.info("Moved segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+ deletedSegmentMoveDestURI.toString());
+ } else {
+ LOGGER.warn("Failed to move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+ deletedSegmentMoveDestURI.toString());
+ }
+ } else {
+ LOGGER.warn("Failed to find local segment file for segment {}", fileToDeleteURI.toString());
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Could not move segment {} from {} to {}", segmentId, fileToDeleteURI.toString(),
+ deletedSegmentMoveDestURI.toString(), e);
+ }
+ }
+
+ private static void segmentDeletion(String segmentId, PinotFS pinotFS, URI fileToDeleteURI) {
+ try {
+ if (pinotFS.delete(fileToDeleteURI, true)) {
+ LOGGER.info("Deleted segment {} from {}", segmentId, fileToDeleteURI.toString());
+ } else {
+ LOGGER.warn("Failed to delete segment {} from {}", segmentId, fileToDeleteURI.toString());
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Could not delete segment {} from {}", segmentId, fileToDeleteURI.toString(), e);
+ }
+ }
+
/**
* Retrieves the URI for segment deletion by checking two possible segment file variants in deep store.
* Looks for the segment file in two formats:
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
index df12b76280..c810151c5c 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
@@ -469,6 +469,40 @@ public class SegmentDeletionManagerTest {
}, 2000L, 10_000L, "Unable to verify table deletion with retention");
}
+ @Test
+ public void testRemoveSegmentsFromStoreInBatch()
+ throws Exception {
+ Map<String, Object> properties = new HashMap<>();
+ properties.put(CommonConstants.Controller.PREFIX_OF_CONFIG_OF_PINOT_FS_FACTORY + ".class",
+ LocalPinotFS.class.getName());
+ PinotFSFactory.init(new PinotConfiguration(properties));
+
+ HelixAdmin helixAdmin = makeHelixAdmin();
+ ZkHelixPropertyStore<ZNRecord> propertyStore = makePropertyStore();
+ File tempDir = Files.createTempDir();
+ tempDir.deleteOnExit();
+ SegmentDeletionManager deletionManager = new SegmentDeletionManager(
+ tempDir.getAbsolutePath(), helixAdmin, CLUSTER_NAME, propertyStore, 7);
+
+ // create table segment files.
+ List<String> segmentsThatShouldBeDeleted = segmentsThatShouldBeDeleted();
+ createTableAndSegmentFiles(tempDir, segmentsThatShouldBeDeleted);
+ final File tableDir = new File(tempDir.getAbsolutePath() + File.separator + TABLE_NAME);
+ final File deletedTableDir = new File(tempDir.getAbsolutePath() + File.separator + "Deleted_Segments"
+ + File.separator + TABLE_NAME);
+
+ deletionManager.removeSegmentsFromStoreInBatch(TABLE_NAME, segmentsThatShouldBeDeleted, 0L);
+
+ TestUtils.waitForCondition(aVoid -> {
+ try {
+ Assert.assertEquals(tableDir.listFiles().length, 0);
+ Assert.assertTrue(!deletedTableDir.exists() || deletedTableDir.listFiles().length == 0);
+ return true;
+ } catch (Throwable t) {
+ return false;
+ }
+ }, 2000L, 10_000L, "Unable to verify table deletion with retention");
+ }
public void createTableAndSegmentFiles(File tempDir, List<String> segmentIds)
throws Exception {
diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
index 3f6656ecaa..c58d9528e0 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
@@ -44,6 +44,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import org.apache.commons.io.FileUtils;
@@ -64,7 +65,7 @@ public class GcsPinotFS extends BasePinotFS {
private static final Logger LOGGER = LoggerFactory.getLogger(GcsPinotFS.class);
// See https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
- private static final int BATCH_LIMIT = 100;
+ private static final int DELETE_BATCH_LIMIT = 100;
private Storage _storage;
@Override
@@ -134,6 +135,28 @@ public class GcsPinotFS extends BasePinotFS {
return delete(new GcsUri(segmentUri), forceDelete);
}
+ @Override
+ public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+ throws IOException {
+ boolean result = true;
+ List<BlobId> blobIds = new ArrayList<>();
+ Iterator<URI> iterator = segmentUris.iterator();
+ while (iterator.hasNext()) {
+ GcsUri gcsUri = new GcsUri(iterator.next());
+ if (existsDirectoryOrBucket(gcsUri)) {
+ result &= delete(gcsUri, forceDelete);
+ } else {
+ blobIds.add(getBlob(gcsUri).getBlobId());
+ if (blobIds.size() >= DELETE_BATCH_LIMIT || !iterator.hasNext()) {
+ List<Boolean> deleted = _storage.delete(blobIds);
+ result &= deleted.stream().allMatch(Boolean::booleanValue);
+ blobIds.clear();
+ }
+ }
+ }
+ return result;
+ }
+
@Override
public boolean doMove(URI srcUri, URI dstUri)
throws IOException {
@@ -428,7 +451,7 @@ public class GcsPinotFS extends BasePinotFS {
for (Blob blob : page.iterateAll()) {
results.add(batch.delete(blob.getBlobId()));
batchSize++;
- if (batchSize >= BATCH_LIMIT) {
+ if (batchSize >= DELETE_BATCH_LIMIT) {
batch.submit();
deleteSucceeded &= results.stream().allMatch(r -> r != null && r.get());
results = new ArrayList<>();
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
index 0339a973e0..5aa240492e 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/main/java/org/apache/pinot/plugin/filesystem/S3PinotFS.java
@@ -66,8 +66,11 @@ import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
@@ -76,6 +79,7 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.MetadataDirective;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -99,6 +103,7 @@ public class S3PinotFS extends BasePinotFS {
public static final String S3_SCHEME = "s3";
public static final String S3A_SCHEME = "s3a";
public static final String SCHEME_SEPARATOR = "://";
+ public static final int DELETE_BATCH_SIZE = 1000;
private S3Client _s3Client;
private boolean _disableAcl;
@@ -390,6 +395,80 @@ public class S3PinotFS extends BasePinotFS {
}
}
+ @Override
+ public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+ throws IOException {
+ boolean deletionResult = true;
+ String prevBucket = null;
+ try {
+ List<ObjectIdentifier> objectsToDelete = new ArrayList<>();
+ LOGGER.info("Deleting URIs {} force {}", segmentUris, forceDelete);
+ // initialize the first bucket
+ if (!segmentUris.isEmpty()) {
+ prevBucket = segmentUris.get(0).getHost();
+ }
+ // Iterate through the URIs and delete them
+ for (URI segmentUri : segmentUris) {
+
+ if (isDirectory(segmentUri)) {
+ if (!forceDelete) {
+ Preconditions.checkState(isEmptyDirectory(segmentUri),
+ "ForceDelete flag is not set and directory '%s' is not empty", segmentUri);
+ }
+
+ // Recursively list files in the directory
+ LOGGER.info("Recursively deleting files in directory {}", segmentUri);
+ List<URI> filesInDir = listFiles(segmentUri);
+ deletionResult &= deleteBatch(filesInDir, forceDelete);
+ } else {
+ String key = sanitizePath(segmentUri.getPath());
+ objectsToDelete.add(ObjectIdentifier.builder().key(key).build());
+ String bucket = segmentUri.getHost();
+
+ // If batch reaches max size, process the batch
+ if (objectsToDelete.size() >= DELETE_BATCH_SIZE || !bucket.equals(prevBucket)) {
+ deletionResult &= processBatch(prevBucket, objectsToDelete);
+ objectsToDelete.clear(); // Clear list for the next batch
+ prevBucket = bucket;
+ }
+ }
+ }
+
+ // Process remaining files in the last batch
+ if (!objectsToDelete.isEmpty()) {
+ deletionResult &= processBatch(prevBucket, objectsToDelete);
+ }
+ return deletionResult;
+ } catch (S3Exception e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private boolean processBatch(String bucket, List<ObjectIdentifier> objectsToDelete) {
+ LOGGER.info("Deleting batch of {} objects", objectsToDelete.size());
+ DeleteObjectsRequest deleteRequest = DeleteObjectsRequest.builder()
+ .bucket(bucket)
+ .delete(Delete.builder().objects(objectsToDelete).build())
+ .build();
+
+ DeleteObjectsResponse deleteResponse = _s3Client.deleteObjects(deleteRequest);
+ LOGGER.info("Failed to delete {} objects", deleteResponse.hasErrors() ? deleteResponse.errors().size() : 0);
+ return deleteResponse.deleted().size() == objectsToDelete.size();
+ }
+
+ private List<URI> listFiles(URI directoryUri)
+ throws IOException {
+ String[] listedFiles = listFiles(directoryUri, true);
+ List<URI> fileUris = new ArrayList<>();
+ for (String filePath : listedFiles) {
+ fileUris.add(URI.create(filePath));
+ }
+ LOGGER.info("Files in directory {}: {}", directoryUri, fileUris);
+ return fileUris;
+ }
+
@Override
public boolean delete(URI segmentUri, boolean forceDelete)
throws IOException {
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
index 8c9ed7f5fe..53bbb96dd4 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
@@ -72,7 +72,7 @@ public class S3PinotFSTest {
@DataProvider(name = "scheme")
public static Object[][] schemes() {
- return new Object[][] { { S3_SCHEME }, { S3A_SCHEME } };
+ return new Object[][]{{S3_SCHEME}, {S3A_SCHEME}};
}
@BeforeClass
@@ -502,15 +502,48 @@ public class S3PinotFSTest {
}
}
+ public void testDeleteBatch()
+ throws IOException {
+ String[] originalFiles = new String[]{"a-delete-batch.txt", "b-delete-batch.txt", "c-delete-batch.txt"};
+ String folderName = "my-files-batch";
+
+ for (String fileName : originalFiles) {
+ createEmptyFile(folderName, fileName);
+ }
+ // Create a sub folder to delete
+ String subFolderName = folderName + DELIMITER + "subfolder";
+ for (String fileName : new String[] {"subfolder-a-delete-batch.txt", "subfolder-b-delete-batch.txt"}) {
+ createEmptyFile(subFolderName, fileName);
+ }
+
+ // Create a list of URIs to delete
+ List<URI> filesToDelete = Arrays.stream(originalFiles)
+ .map(fileName -> URI.create(String.format(FILE_FORMAT, S3_SCHEME, BUCKET, folderName + DELIMITER + fileName)))
+ .collect(Collectors.toList());
+ filesToDelete.add(URI.create(String.format(FILE_FORMAT, S3_SCHEME, BUCKET, subFolderName)));
+
+ boolean deleteResult = _s3PinotFS.deleteBatch(filesToDelete, true);
+
+ Assert.assertTrue(deleteResult);
+
+ ListObjectsV2Response listObjectsV2Response =
+ _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true));
+ String[] actualResponse =
+ listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("delete-batch"))
+ .toArray(String[]::new);
+
+ Assert.assertEquals(actualResponse.length, 0);
+ }
+
@DataProvider(name = "storageClasses")
public Object[][] createStorageClasses() {
- return new Object[][] {
- { null, S3_SCHEME },
- { StorageClass.STANDARD, S3_SCHEME },
- { StorageClass.INTELLIGENT_TIERING, S3_SCHEME },
- { null, S3A_SCHEME },
- { StorageClass.STANDARD, S3A_SCHEME },
- { StorageClass.INTELLIGENT_TIERING, S3A_SCHEME },
+ return new Object[][]{
+ {null, S3_SCHEME},
+ {StorageClass.STANDARD, S3_SCHEME},
+ {StorageClass.INTELLIGENT_TIERING, S3_SCHEME},
+ {null, S3A_SCHEME},
+ {StorageClass.STANDARD, S3A_SCHEME},
+ {StorageClass.INTELLIGENT_TIERING, S3A_SCHEME},
};
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
index bd76bdf740..8d2724f25a 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/BasePinotFS.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +31,16 @@ import org.slf4j.LoggerFactory;
public abstract class BasePinotFS implements PinotFS {
private static final Logger LOGGER = LoggerFactory.getLogger(BasePinotFS.class);
+ @Override
+ public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+ throws IOException {
+ boolean result = true;
+ for (URI segmentUri : segmentUris) {
+ result &= delete(segmentUri, forceDelete);
+ }
+ return result;
+ }
+
@Override
public boolean move(URI srcUri, URI dstUri, boolean overwrite)
throws IOException {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
index d9fc002c7e..de29fe4cb8 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/NoClosePinotFS.java
@@ -50,6 +50,12 @@ public class NoClosePinotFS implements PinotFS {
return _delegate.delete(segmentUri, forceDelete);
}
+ @Override
+ public boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+ throws IOException {
+ return _delegate.deleteBatch(segmentUris, forceDelete);
+ }
+
@Override
public boolean move(URI srcUri, URI dstUri, boolean overwrite)
throws IOException {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
index 286a02c7d4..863df527f9 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/PinotFS.java
@@ -70,6 +70,16 @@ public interface PinotFS extends Closeable, Serializable {
boolean delete(URI segmentUri, boolean forceDelete)
throws IOException;
+ /**
+ * Deletes the files at the locations provided. If the segmentUri is a directory, it will delete the entire directory.
+ * @param segmentUris List of URIs of the segments
+ * @param forceDelete true if we want the uri and any sub-uris to always be deleted, false if we want delete to fail
+ * @return true if delete is successful else false
+ * @throws IOException on IO failure, e.g Uri is not present or not valid
+ */
+ boolean deleteBatch(List<URI> segmentUris, boolean forceDelete)
+ throws IOException;
+
/**
* Moves the file or directory from the src to dst. Does not keep the original file. If the dst has parent directories
* that haven't been created, this method will create all the necessary parent directories.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org