You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by "Jackie-Jiang (via GitHub)" <gi...@apache.org> on 2023/04/27 19:32:52 UTC

[GitHub] [pinot] Jackie-Jiang commented on a diff in pull request #10639: changing the dedup store to become pluggable

Jackie-Jiang commented on code in PR #10639:
URL: https://github.com/apache/pinot/pull/10639#discussion_r1179596748


##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/dedup/PartitionDedupMetadataManager.java:
##########
@@ -39,46 +44,73 @@ public class PartitionDedupMetadataManager {
   private final List<String> _primaryKeyColumns;
   private final int _partitionId;
   private final ServerMetrics _serverMetrics;
-  private final HashFunction _hashFunction;
-
-  @VisibleForTesting
-  final ConcurrentHashMap<Object, IndexSegment> _primaryKeyToSegmentMap = new ConcurrentHashMap<>();
+  private final DedupConfig _dedupConfig;
+  @VisibleForTesting final LocalKeyValueStore _keyValueStore;
 
   public PartitionDedupMetadataManager(String tableNameWithType, List<String> primaryKeyColumns, int partitionId,
-      ServerMetrics serverMetrics, HashFunction hashFunction) {
+      ServerMetrics serverMetrics, DedupConfig dedupConfig) {
     _tableNameWithType = tableNameWithType;
     _primaryKeyColumns = primaryKeyColumns;
     _partitionId = partitionId;
     _serverMetrics = serverMetrics;
-    _hashFunction = hashFunction;
+    _dedupConfig = dedupConfig;
+    try {
+      byte[] id = (tableNameWithType + "@" + partitionId).getBytes();
+      _keyValueStore = StringUtils.isEmpty(dedupConfig.getKeyStore())
+              ? new ConcurrentHashMapKeyValueStore()
+              : PluginManager.get().createInstance(
+                      dedupConfig.getKeyStore(),
+                      new Class[]{byte[].class},
+                      new Object[]{id}
+              );
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
   }
 
   public void addSegment(IndexSegment segment) {
-    // Add all PKs to _primaryKeyToSegmentMap
     Iterator<PrimaryKey> primaryKeyIterator = getPrimaryKeyIterator(segment);
+    byte[] serializedSegment = serializeSegment(segment);
+    List<Pair<byte[], byte[]>> keyValues = new ArrayList<>();
     while (primaryKeyIterator.hasNext()) {
       PrimaryKey pk = primaryKeyIterator.next();
-      _primaryKeyToSegmentMap.put(HashUtils.hashPrimaryKey(pk, _hashFunction), segment);
+      byte[] serializedPrimaryKey = serializePrimaryKey(HashUtils.hashPrimaryKey(pk, _dedupConfig.getHashFunction()));
+      keyValues.add(Pair.of(serializedPrimaryKey, serializedSegment));
     }
+    _keyValueStore.putBatch(keyValues);
     _serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId, ServerGauge.DEDUP_PRIMARY_KEYS_COUNT,
-        _primaryKeyToSegmentMap.size());
+        _keyValueStore.getKeyCount());
+  }
+
+  @VisibleForTesting
+  static byte[] serializeSegment(IndexSegment segment) {
+    return segment.getSegmentName().getBytes();
+  }
+
+  @VisibleForTesting
+  static byte[] serializePrimaryKey(Object pk) {
+    if (pk instanceof PrimaryKey) {
+      return ((PrimaryKey) pk).asBytes();
+    }
+    if (pk instanceof ByteArray) {
+      return ((ByteArray) pk).getBytes();
+    }
+    throw new RuntimeException("Unsupported pk class: " + pk);
   }
 
   public void removeSegment(IndexSegment segment) {
     // TODO(saurabh): Explain reload scenario here
     Iterator<PrimaryKey> primaryKeyIterator = getPrimaryKeyIterator(segment);
+    byte[] segmentBytes = serializeSegment(segment);
     while (primaryKeyIterator.hasNext()) {
       PrimaryKey pk = primaryKeyIterator.next();
-      _primaryKeyToSegmentMap.compute(HashUtils.hashPrimaryKey(pk, _hashFunction), (primaryKey, currentSegment) -> {
-        if (currentSegment == segment) {
-          return null;
-        } else {
-          return currentSegment;
-        }
-      });
+      byte[] pkBytes = serializePrimaryKey(pk);
+      if (Objects.deepEquals(_keyValueStore.get(pkBytes), segmentBytes)) {

Review Comment:
   This can potentially cause race condition when the segment is added again during the segment removal, and that's why we perform the reference check before



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org