You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2021/09/13 18:17:54 UTC
[pinot] branch fix-retention-for-segment-lineage created (now
7656e3a)
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a change to branch fix-retention-for-segment-lineage
in repository https://gitbox.apache.org/repos/asf/pinot.git.
at 7656e3a Fix retention for cleaning up segment lineage
This branch includes the following new commits:
new 7656e3a Fix retention for cleaning up segment lineage
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[pinot] 01/01: Fix retention for cleaning up segment lineage
Posted by jl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch fix-retention-for-segment-lineage
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 7656e3a50fd6d9e2b3544b892540a54fc24e8d48
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Mon Sep 13 11:17:19 2021 -0700
Fix retention for cleaning up segment lineage
---
.../helix/core/retention/RetentionManager.java | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
index b9b0d09..79da07f 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/retention/RetentionManager.java
@@ -90,7 +90,16 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
}
private void manageRetentionForTable(String tableNameWithType) {
+ // Manage table retention except segment lineage cleanup.
+ // The reason of separating them is that REFRESH only table will be skipped in the first part,
+ // whereas the segment lineage cleanup needs to be handled.
+ manageRetentionExceptSegmentLineageCleanup(tableNameWithType);
+ // Delete segments based on segment lineage and clean up segment lineage metadata.
+ manageSegmentLineageCleanupForTable(tableNameWithType);
+ }
+
+ private void manageRetentionExceptSegmentLineageCleanup(String tableNameWithType) {
// Build retention strategy from table config
TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType);
if (tableConfig == null) {
@@ -123,9 +132,6 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
} else {
manageRetentionForRealtimeTable(tableNameWithType, retentionStrategy);
}
-
- // Delete segments based on segment lineage and clean up segment lineage metadata
- manageSegmentLineageCleanupForTable(tableNameWithType);
}
private void manageRetentionForOfflineTable(String offlineTableName, RetentionStrategy retentionStrategy) {
@@ -202,6 +208,8 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
if (segmentLineageZNRecord == null) {
return true;
}
+ LOGGER.info("Start cleaning up segment lineage for table: {}", tableNameWithType);
+ long cleanupStartTime = System.currentTimeMillis();
SegmentLineage segmentLineage = SegmentLineage.fromZNRecord(segmentLineageZNRecord);
int expectedVersion = segmentLineageZNRecord.getVersion();
@@ -246,8 +254,12 @@ public class RetentionManager extends ControllerPeriodicTask<Void> {
.writeSegmentLineage(_pinotHelixResourceManager.getPropertyStore(), segmentLineage, expectedVersion)) {
// Delete segments based on the segment lineage
_pinotHelixResourceManager.deleteSegments(tableNameWithType, segmentsToDelete);
+ LOGGER.info("Finished cleaning up segment lineage for table: {} in {}ms", tableNameWithType,
+ (System.currentTimeMillis() - cleanupStartTime));
return true;
} else {
+ LOGGER.info("Failed to write segment lineage back when cleaning up segment lineage for table: {}",
+ tableNameWithType);
return false;
}
});
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org