You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2026/03/20 23:11:48 UTC
(pinot) branch master updated: [Upsert] Optimize reverting preveious keys (#17918)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 550afb77119 [Upsert] Optimize reverting preveious keys (#17918)
550afb77119 is described below
commit 550afb77119492bd612d07011d7f3051d1a4a782
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Fri Mar 20 16:11:42 2026 -0700
[Upsert] Optimize reverting preveious keys (#17918)
---
.../ConcurrentMapPartitionUpsertMetadataManager.java | 15 +++++++--------
...artitionUpsertMetadataManagerForConsistentDeletes.java | 14 ++++++--------
2 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManager.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManager.java
index be9c0b17307..832f2ac5ebb 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManager.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManager.java
@@ -19,7 +19,6 @@
package org.apache.pinot.segment.local.upsert;
import com.google.common.annotations.VisibleForTesting;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -233,10 +232,9 @@ public class ConcurrentMapPartitionUpsertMetadataManager extends BasePartitionUp
int docId = primaryKeyEntry.getKey();
_primaryKeyToRecordLocationMap.computeIfPresent(HashUtils.hashPrimaryKey(primaryKey, _hashFunction),
(pk, recordLocation) -> {
- RecordLocation prevLocation = _previousKeyToRecordLocationMap.get(pk);
if (recordLocation.getSegment() == segment) {
+ RecordLocation prevLocation = _previousKeyToRecordLocationMap.remove(pk);
if (prevLocation == null) {
- _previousKeyToRecordLocationMap.remove(pk);
return null;
}
// Revert to previous segment location
@@ -249,19 +247,20 @@ public class ConcurrentMapPartitionUpsertMetadataManager extends BasePartitionUp
RecordInfo recordInfo = recordInfoReader.getRecordInfo(prevDocId);
replaceDocId(prevSegment, prevValidDocIds, prevSegment.getQueryableDocIds(), segment, docId,
prevDocId, recordInfo);
- _previousKeyToRecordLocationMap.remove(pk);
return prevLocation;
- } catch (IOException e) {
- _logger.warn("Failed to revert to previous segment: {}, removing key", prevSegment.getSegmentName(),
+ } catch (Exception e) {
+ _logger.error("Failed to revert to previous segment: {}, removing key", prevSegment.getSegmentName(),
e);
- _previousKeyToRecordLocationMap.remove(pk);
return null;
}
} else {
- _previousKeyToRecordLocationMap.remove(pk);
+ // Should not happen
+ _logger.error("Failed to find valid doc ids in previous segment: {}, removing key",
+ prevSegment.getSegmentName());
return null;
}
} else if (recordLocation.getSegment() instanceof ImmutableSegmentImpl) {
+ // The consuming segment's key is in a different immutable segment
_previousKeyToRecordLocationMap.remove(pk);
} else {
_logger.warn(
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes.java
index d21120a0450..038eba8e6af 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes.java
@@ -20,7 +20,6 @@ package org.apache.pinot.segment.local.upsert;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -360,10 +359,9 @@ public class ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes
int docId = primaryKeyEntry.getKey();
_primaryKeyToRecordLocationMap.computeIfPresent(HashUtils.hashPrimaryKey(primaryKey, _hashFunction),
(pk, recordLocation) -> {
- RecordLocation prevLocation = _previousKeyToRecordLocationMap.get(pk);
if (recordLocation.getSegment() == segment) {
+ RecordLocation prevLocation = _previousKeyToRecordLocationMap.remove(pk);
if (prevLocation == null) {
- _previousKeyToRecordLocationMap.remove(pk);
return null;
}
// Revert to previous segment location
@@ -376,21 +374,21 @@ public class ConcurrentMapPartitionUpsertMetadataManagerForConsistentDeletes
RecordInfo recordInfo = recordInfoReader.getRecordInfo(prevDocId);
replaceDocId(prevSegment, prevValidDocIds, prevSegment.getQueryableDocIds(), segment, docId,
prevDocId, recordInfo);
- _previousKeyToRecordLocationMap.remove(pk);
if (!uniquePrimaryKeys.add(pk)) {
return prevLocation;
}
return new RecordLocation(prevLocation.getSegment(), prevLocation.getDocId(),
prevLocation.getComparisonValue(),
RecordLocation.decrementSegmentCount(prevLocation.getDistinctSegmentCount()));
- } catch (IOException e) {
- _logger.warn("Failed to revert to previous segment: {}, removing key", prevSegment.getSegmentName(),
+ } catch (Exception e) {
+ _logger.error("Failed to revert to previous segment: {}, removing key", prevSegment.getSegmentName(),
e);
- _previousKeyToRecordLocationMap.remove(pk);
return null;
}
} else {
- _previousKeyToRecordLocationMap.remove(pk);
+ // Should not happen
+ _logger.error("Failed to find valid doc ids in previous segment: {}, removing key",
+ prevSegment.getSegmentName());
return null;
}
} else if (recordLocation.getSegment() instanceof ImmutableSegmentImpl) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org