You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2022/02/20 20:32:31 UTC
[pinot] branch master updated: adding retention period to segment delete REST API (#8122)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4f17ede adding retention period to segment delete REST API (#8122)
4f17ede is described below
commit 4f17ede26c3a5c92dea5106654e685a2a411287d
Author: Rong Rong <wa...@gmail.com>
AuthorDate: Sun Feb 20 12:31:59 2022 -0800
adding retention period to segment delete REST API (#8122)
Add deletion with retention period overwrite for segment deletion
---
.../api/resources/PinotSegmentRestletResource.java | 22 +++++++++++++++-------
.../helix/core/PinotHelixResourceManager.java | 18 +++++++++++++++---
.../helix/core/SegmentDeletionManager.java | 8 ++++++--
3 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
index 7078e60..4f2d31d 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
@@ -571,12 +571,14 @@ public class PinotSegmentRestletResource {
@ApiOperation(value = "Delete a segment", notes = "Delete a segment")
public SuccessResponse deleteSegment(
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
- @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName) {
+ @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") @Encoded String segmentName,
+ @ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
+ + "delete segments without retention") @QueryParam("retention") String retentionPeriod) {
segmentName = URIUtils.decode(segmentName);
TableType tableType = SegmentName.isRealtimeSegmentName(segmentName) ? TableType.REALTIME : TableType.OFFLINE;
String tableNameWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
- deleteSegmentsInternal(tableNameWithType, Collections.singletonList(segmentName));
+ deleteSegmentsInternal(tableNameWithType, Collections.singletonList(segmentName), retentionPeriod);
return new SuccessResponse("Segment deleted");
}
@@ -587,14 +589,17 @@ public class PinotSegmentRestletResource {
@ApiOperation(value = "Delete all segments", notes = "Delete all segments")
public SuccessResponse deleteAllSegments(
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
- @ApiParam(value = "OFFLINE|REALTIME", required = true) @QueryParam("type") String tableTypeStr) {
+ @ApiParam(value = "OFFLINE|REALTIME", required = true) @QueryParam("type") String tableTypeStr,
+ @ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
+ + "delete segments without retention") @QueryParam("retention") String retentionPeriod) {
TableType tableType = Constants.validateTableType(tableTypeStr);
if (tableType == null) {
throw new ControllerApplicationException(LOGGER, "Table type must not be null", Status.BAD_REQUEST);
}
String tableNameWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
- deleteSegmentsInternal(tableNameWithType, _pinotHelixResourceManager.getSegmentsFor(tableNameWithType, false));
+ deleteSegmentsInternal(tableNameWithType, _pinotHelixResourceManager.getSegmentsFor(tableNameWithType, false),
+ retentionPeriod);
return new SuccessResponse("All segments of table " + tableNameWithType + " deleted");
}
@@ -607,6 +612,8 @@ public class PinotSegmentRestletResource {
notes = "Delete the segments in the JSON array payload")
public SuccessResponse deleteSegments(
@ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
+ @ApiParam(value = "Retention period for the deleted segments (e.g. 12h, 3d); Using 0d or -1d will instantly "
+ + "delete segments without retention") @QueryParam("retention") String retentionPeriod,
List<String> segments) {
int numSegments = segments.size();
if (numSegments == 0) {
@@ -622,7 +629,7 @@ public class PinotSegmentRestletResource {
TableType tableType = isRealtimeSegment ? TableType.REALTIME : TableType.OFFLINE;
String tableNameWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
- deleteSegmentsInternal(tableNameWithType, segments);
+ deleteSegmentsInternal(tableNameWithType, segments, retentionPeriod);
if (numSegments <= 5) {
return new SuccessResponse("Deleted segments: " + segments + " from table: " + tableNameWithType);
} else {
@@ -630,8 +637,9 @@ public class PinotSegmentRestletResource {
}
}
- private void deleteSegmentsInternal(String tableNameWithType, List<String> segments) {
- PinotResourceManagerResponse response = _pinotHelixResourceManager.deleteSegments(tableNameWithType, segments);
+ private void deleteSegmentsInternal(String tableNameWithType, List<String> segments, String retentionPeriod) {
+ PinotResourceManagerResponse response = _pinotHelixResourceManager.deleteSegments(tableNameWithType, segments,
+ retentionPeriod);
if (!response.isSuccessful()) {
throw new ControllerApplicationException(LOGGER,
"Failed to delete segments from table: " + tableNameWithType + ", error message: " + response.getMessage(),
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index df64e2d..ce82e5f 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -133,6 +133,7 @@ import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.BrokerResourc
import org.apache.pinot.spi.utils.CommonConstants.Helix.StateModel.SegmentStateModel;
import org.apache.pinot.spi.utils.CommonConstants.Server;
import org.apache.pinot.spi.utils.IngestionConfigUtils;
+import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.apache.pinot.spi.utils.retry.RetryPolicy;
@@ -694,21 +695,32 @@ public class PinotHelixResourceManager {
return ZKMetadataProvider.getSegmentsZKMetadata(_propertyStore, tableNameWithType);
}
+ public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames) {
+ return deleteSegments(tableNameWithType, segmentNames, null);
+ }
+
/**
* Delete a list of segments from ideal state and remove them from the local storage.
*
* @param tableNameWithType Table name with type suffix
* @param segmentNames List of names of segment to be deleted
+ * @param retentionPeriod The retention period of the deleted segments.
* @return Request response
*/
- public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames) {
+ public synchronized PinotResourceManagerResponse deleteSegments(String tableNameWithType, List<String> segmentNames,
+ @Nullable String retentionPeriod) {
try {
LOGGER.info("Trying to delete segments: {} from table: {} ", segmentNames, tableNameWithType);
Preconditions.checkArgument(TableNameBuilder.isTableResource(tableNameWithType),
"Table name: %s is not a valid table name with type suffix", tableNameWithType);
HelixHelper.removeSegmentsFromIdealState(_helixZkManager, tableNameWithType, segmentNames);
- TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType);
- _segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames, tableConfig);
+ if (retentionPeriod != null) {
+ _segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames,
+ TimeUtils.convertPeriodToMillis(retentionPeriod));
+ } else {
+ TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType);
+ _segmentDeletionManager.deleteSegments(tableNameWithType, segmentNames, tableConfig);
+ }
return PinotResourceManagerResponse.success("Segment " + segmentNames + " deleted");
} catch (final Exception e) {
LOGGER.error("Caught exception while deleting segment: {} from table: {}", segmentNames, tableNameWithType, e);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
index 35de4e7..43372e2 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/SegmentDeletionManager.java
@@ -103,12 +103,16 @@ public class SegmentDeletionManager {
}
public void deleteSegments(String tableName, Collection<String> segmentIds) {
- deleteSegments(tableName, segmentIds, null);
+ deleteSegments(tableName, segmentIds, (Long) null);
}
public void deleteSegments(String tableName, Collection<String> segmentIds,
@Nullable TableConfig tableConfig) {
- Long deletedSegmentsRetentionMs = getRetentionMsFromTableConfig(tableConfig);
+ deleteSegments(tableName, segmentIds, getRetentionMsFromTableConfig(tableConfig));
+ }
+
+ public void deleteSegments(String tableName, Collection<String> segmentIds,
+ @Nullable Long deletedSegmentsRetentionMs) {
deleteSegmentsWithDelay(tableName, segmentIds, deletedSegmentsRetentionMs, DEFAULT_DELETION_DELAY_SECONDS);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org