You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ma...@apache.org on 2024/06/26 20:49:00 UTC
(kafka) branch trunk updated: KAFKA-16751: Implemented release acquired records functionality in SharePartitionManager (#16446)
This is an automated email from the ASF dual-hosted git repository.
manikumar pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 399949ebcf3 KAFKA-16751: Implemented release acquired records functionality in SharePartitionManager (#16446)
399949ebcf3 is described below
commit 399949ebcf3815e088a62495996100f27f637ec0
Author: Abhinav Dixit <ad...@confluent.io>
AuthorDate: Thu Jun 27 02:18:54 2024 +0530
KAFKA-16751: Implemented release acquired records functionality in SharePartitionManager (#16446)
About
Implemented releaseAcquiredRecords functionality in SharePartitionManager which will act as a bridge between the call from KafkaApis to SharePartition for releasing the acquired records when a share session gets closed.
Testing
The added function has been tested with unit tests.
Reviewers: Andrew Schofield <as...@confluent.io>, Manikumar Reddy <ma...@gmail.com>, Apoorv Mittal <ap...@gmail.com>
---
.../kafka/server/share/SharePartitionManager.java | 29 ++++-
.../server/share/SharePartitionManagerTest.java | 125 +++++++++++++++++++++
2 files changed, 150 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/kafka/server/share/SharePartitionManager.java b/core/src/main/java/kafka/server/share/SharePartitionManager.java
index 9d212165ad4..a18d8dbe854 100644
--- a/core/src/main/java/kafka/server/share/SharePartitionManager.java
+++ b/core/src/main/java/kafka/server/share/SharePartitionManager.java
@@ -276,11 +276,32 @@ public class SharePartitionManager implements AutoCloseable {
) {
log.trace("Release acquired records request for topicIdPartitions: {} with groupId: {}",
topicIdPartitions, groupId);
+ Map<TopicIdPartition, CompletableFuture<Errors>> futuresMap = new HashMap<>();
+ topicIdPartitions.forEach(topicIdPartition -> {
+ SharePartition sharePartition = partitionCacheMap.get(sharePartitionKey(groupId, topicIdPartition));
+ if (sharePartition == null) {
+ log.error("No share partition found for groupId {} topicPartition {} while releasing acquired topic partitions", groupId, topicIdPartition);
+ futuresMap.put(topicIdPartition, CompletableFuture.completedFuture(Errors.UNKNOWN_TOPIC_OR_PARTITION));
+ } else {
+ CompletableFuture<Errors> future = sharePartition.releaseAcquiredRecords(memberId).thenApply(throwable -> {
+ if (throwable.isPresent()) {
+ return Errors.forException(throwable.get());
+ }
+ return Errors.NONE;
+ });
+ futuresMap.put(topicIdPartition, future);
+ }
+ });
- CompletableFuture<Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData>> future = new CompletableFuture<>();
- future.completeExceptionally(new UnsupportedOperationException("Not implemented yet"));
-
- return future;
+ CompletableFuture<Void> allFutures = CompletableFuture.allOf(
+ futuresMap.values().toArray(new CompletableFuture[futuresMap.size()]));
+ return allFutures.thenApply(v -> {
+ Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData> result = new HashMap<>();
+ futuresMap.forEach((topicIdPartition, future) -> result.put(topicIdPartition, new ShareAcknowledgeResponseData.PartitionData()
+ .setPartitionIndex(topicIdPartition.partition())
+ .setErrorCode(future.join().code())));
+ return result;
+ });
}
/**
diff --git a/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java b/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
index cd232b19d0e..bf14d25a303 100644
--- a/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
+++ b/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
@@ -23,9 +23,11 @@ import org.apache.kafka.common.TopicIdPartition;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.compress.Compression;
+import org.apache.kafka.common.errors.InvalidRecordStateException;
import org.apache.kafka.common.errors.InvalidRequestException;
import org.apache.kafka.common.errors.InvalidShareSessionEpochException;
import org.apache.kafka.common.errors.ShareSessionNotFoundException;
+import org.apache.kafka.common.message.ShareAcknowledgeResponseData;
import org.apache.kafka.common.message.ShareFetchResponseData;
import org.apache.kafka.common.message.ShareFetchResponseData.PartitionData;
import org.apache.kafka.common.protocol.ApiKeys;
@@ -55,6 +57,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import java.util.ArrayList;
@@ -1239,6 +1242,128 @@ public class SharePartitionManagerTest {
Mockito.verify(persister, times(1)).stop();
}
+ @Test
+ public void testReleaseAcquiredRecordsSuccess() {
+ String groupId = "grp";
+ String memberId = Uuid.randomUuid().toString();
+
+ TopicIdPartition tp1 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+ TopicIdPartition tp2 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("bar", 2));
+ TopicIdPartition tp3 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("baz", 4));
+
+ SharePartition sp1 = mock(SharePartition.class);
+ SharePartition sp2 = mock(SharePartition.class);
+ when(sp1.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+ when(sp2.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(
+ Optional.of(new InvalidRecordStateException("Unable to release acquired records for the batch"))
+ ));
+ Map<SharePartitionManager.SharePartitionKey, SharePartition> partitionCacheMap = new HashMap<>();
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp1), sp1);
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp2), sp2);
+
+ SharePartitionManager sharePartitionManager = SharePartitionManagerBuilder.builder()
+ .withPartitionCacheMap(partitionCacheMap).build();
+ CompletableFuture<Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData>> resultFuture =
+ sharePartitionManager.releaseAcquiredRecords(groupId, memberId, Arrays.asList(tp1, tp2, tp3));
+ Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData> result = resultFuture.join();
+ assertEquals(3, result.size());
+ assertTrue(result.containsKey(tp1));
+ assertTrue(result.containsKey(tp2));
+ assertTrue(result.containsKey(tp3));
+ assertEquals(0, result.get(tp1).partitionIndex());
+ assertEquals(Errors.NONE.code(), result.get(tp1).errorCode());
+ assertEquals(2, result.get(tp2).partitionIndex());
+ assertEquals(Errors.INVALID_RECORD_STATE.code(), result.get(tp2).errorCode());
+ // tp3 was not a part of partitionCacheMap.
+ assertEquals(4, result.get(tp3).partitionIndex());
+ assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code(), result.get(tp3).errorCode());
+ }
+
+ @Test
+ public void testReleaseAcquiredRecordsWithIncorrectGroupId() {
+ String memberId = Uuid.randomUuid().toString();
+
+ TopicIdPartition tp1 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+
+ SharePartition sp1 = mock(SharePartition.class);
+ when(sp1.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+ Map<SharePartitionManager.SharePartitionKey, SharePartition> partitionCacheMap = new HashMap<>();
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey("grp", tp1), sp1);
+
+ SharePartitionManager sharePartitionManager = SharePartitionManagerBuilder.builder()
+ .withPartitionCacheMap(partitionCacheMap).build();
+
+ // Calling releaseAcquiredRecords with incorrect groupId.
+ CompletableFuture<Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData>> resultFuture =
+ sharePartitionManager.releaseAcquiredRecords("grp-2", memberId, Collections.singletonList(tp1));
+ Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData> result = resultFuture.join();
+ assertEquals(1, result.size());
+ assertTrue(result.containsKey(tp1));
+ assertEquals(0, result.get(tp1).partitionIndex());
+ assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code(), result.get(tp1).errorCode());
+ }
+
+ @Test
+ public void testReleaseAcquiredRecordsWithIncorrectMemberId() {
+ String groupId = "grp";
+ String memberId = Uuid.randomUuid().toString();
+
+ TopicIdPartition tp1 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+ TopicIdPartition tp2 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("bar", 2));
+
+ SharePartition sp1 = mock(SharePartition.class);
+ SharePartition sp2 = mock(SharePartition.class);
+ // When incorrect memberId is passed, releaseAcquiredRecords should return an empty Optional.
+ when(sp1.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+ // When incorrect memberId is passed, releaseAcquiredRecords should return an empty Optional.
+ when(sp2.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+
+ Map<SharePartitionManager.SharePartitionKey, SharePartition> partitionCacheMap = new HashMap<>();
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp1), sp1);
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp2), sp2);
+
+ SharePartitionManager sharePartitionManager = SharePartitionManagerBuilder.builder()
+ .withPartitionCacheMap(partitionCacheMap).build();
+
+ CompletableFuture<Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData>> resultFuture =
+ sharePartitionManager.releaseAcquiredRecords(groupId, memberId, Arrays.asList(tp1, tp2));
+ Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData> result = resultFuture.join();
+ assertEquals(2, result.size());
+ assertTrue(result.containsKey(tp1));
+ assertTrue(result.containsKey(tp2));
+ assertEquals(0, result.get(tp1).partitionIndex());
+ assertEquals(Errors.NONE.code(), result.get(tp1).errorCode());
+ assertEquals(2, result.get(tp2).partitionIndex());
+ assertEquals(Errors.NONE.code(), result.get(tp2).errorCode());
+ }
+
+ @Test
+ public void testReleaseAcquiredRecordsWithEmptyTopicPartitions() {
+ String groupId = "grp";
+ String memberId = Uuid.randomUuid().toString();
+
+ TopicIdPartition tp1 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+ TopicIdPartition tp2 = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("bar", 2));
+
+ SharePartition sp1 = mock(SharePartition.class);
+ SharePartition sp2 = mock(SharePartition.class);
+ when(sp1.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+ when(sp2.releaseAcquiredRecords(ArgumentMatchers.eq(memberId))).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
+
+ Map<SharePartitionManager.SharePartitionKey, SharePartition> partitionCacheMap = new HashMap<>();
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp1), sp1);
+ partitionCacheMap.put(new SharePartitionManager.SharePartitionKey(groupId, tp2), sp2);
+
+ SharePartitionManager sharePartitionManager = SharePartitionManagerBuilder.builder()
+ .withPartitionCacheMap(partitionCacheMap).build();
+
+ // Pass an empty list of TopicIdPartitions to releaseAcquiredRecords. This should return an empty map.
+ CompletableFuture<Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData>> resultFuture =
+ sharePartitionManager.releaseAcquiredRecords(groupId, memberId, Collections.emptyList());
+ Map<TopicIdPartition, ShareAcknowledgeResponseData.PartitionData> result = resultFuture.join();
+ assertEquals(0, result.size());
+ }
+
private ShareFetchResponseData.PartitionData noErrorShareFetchResponse() {
return new ShareFetchResponseData.PartitionData().setPartitionIndex(0);
}