You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2024/02/02 07:05:52 UTC
(pinot) branch master updated: Add segment metadata deletion code to segment deletion path (#12350)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 041e04078f Add segment metadata deletion code to segment deletion path (#12350)
041e04078f is described below
commit 041e04078f5a94fca92c805a8db8fdf1f904a985
Author: swaminathanmanish <12...@users.noreply.github.com>
AuthorDate: Thu Feb 1 23:05:46 2024 -0800
Add segment metadata deletion code to segment deletion path (#12350)
---
.../helix/core/SegmentDeletionManager.java | 27 ++++++++++++++++++----
.../core/util/SegmentDeletionManagerTest.java | 4 ++++
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index 8a0af22a2c..5a7012657e 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -20,6 +20,7 @@ package org.apache.pinot.controller.helix.core;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -46,6 +47,7 @@ import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.core.segment.processing.lifecycle.PinotSegmentLifecycleEventListenerManager;
import org.apache.pinot.core.segment.processing.lifecycle.impl.SegmentDeletionEventDetails;
+import org.apache.pinot.segment.local.utils.SegmentPushUtils;
import org.apache.pinot.spi.config.table.SegmentsValidationAndRetentionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.filesystem.PinotFS;
@@ -108,8 +110,7 @@ public class SegmentDeletionManager {
deleteSegments(tableName, segmentIds, (Long) null);
}
- public void deleteSegments(String tableName, Collection<String> segmentIds,
- @Nullable TableConfig tableConfig) {
+ public void deleteSegments(String tableName, Collection<String> segmentIds, @Nullable TableConfig tableConfig) {
deleteSegments(tableName, segmentIds, getRetentionMsFromTableConfig(tableConfig));
}
@@ -123,8 +124,7 @@ public class SegmentDeletionManager {
_executorService.schedule(new Runnable() {
@Override
public void run() {
- deleteSegmentFromPropertyStoreAndLocal(tableName, segmentIds, deletedSegmentsRetentionMs,
- deletionDelaySeconds);
+ deleteSegmentFromPropertyStoreAndLocal(tableName, segmentIds, deletedSegmentsRetentionMs, deletionDelaySeconds);
}
}, deletionDelaySeconds, TimeUnit.SECONDS);
}
@@ -213,6 +213,22 @@ public class SegmentDeletionManager {
}
}
+ private void deleteSegmentMetadataFromStore(PinotFS pinotFS, URI segmentFileUri, String segmentId) {
+ // Check if segment metadata exists in remote store and delete it.
+ // URI is generated from segment's location and segment name
+ try {
+ URI segmentMetadataUri = SegmentPushUtils.generateSegmentMetadataURI(segmentFileUri.toString(), segmentId);
+ if (pinotFS.exists(segmentMetadataUri)) {
+ LOGGER.info("Deleting segment metadata {} from {}", segmentId, segmentMetadataUri);
+ pinotFS.delete(segmentMetadataUri, true);
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Could not delete segment metadata {} from {}", segmentId, segmentFileUri, e);
+ } catch (URISyntaxException e) {
+ LOGGER.warn("Could not parse segment uri {}", segmentFileUri, e);
+ }
+ }
+
protected void removeSegmentFromStore(String tableNameWithType, String segmentId,
@Nullable Long deletedSegmentsRetentionMs) {
if (_dataDir != null) {
@@ -221,6 +237,9 @@ public class SegmentDeletionManager {
String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType);
URI fileToDeleteURI = URIUtils.getUri(_dataDir, rawTableName, URIUtils.encode(segmentId));
PinotFS pinotFS = PinotFSFactory.create(fileToDeleteURI.getScheme());
+ // Segment metadata in remote store is an optimization, to avoid downloading segment to parse metadata.
+ // This is catch all clean up to ensure that metadata is removed from deep store.
+ deleteSegmentMetadataFromStore(pinotFS, fileToDeleteURI, segmentId);
if (retentionMs <= 0) {
// delete the segment file instantly if retention is set to zero
try {
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
index 4b585f35fc..e600643934 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/util/SegmentDeletionManagerTest.java
@@ -45,6 +45,7 @@ import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.filesystem.LocalPinotFS;
import org.apache.pinot.spi.filesystem.PinotFSFactory;
+import org.apache.pinot.spi.ingestion.batch.spec.Constants;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.util.TestUtils;
import org.joda.time.DateTime;
@@ -357,6 +358,9 @@ public class SegmentDeletionManagerTest {
tableDir.mkdir();
for (String segmentId : segmentIds) {
createTestFileWithAge(tableDir.getAbsolutePath() + File.separator + segmentId, 0);
+ // Create segment metadata file
+ createTestFileWithAge(
+ tableDir.getAbsolutePath() + File.separator + segmentId + Constants.METADATA_TAR_GZ_FILE_EXT, 0);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org