You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by mi...@apache.org on 2026/09/21 13:49:22 UTC
(kafka) branch 4.3 updated: MINOR: Moving to atomic update of share sessions on initial epoch (#23510)
This is an automated email from the ASF dual-hosted git repository.
apoorvmittal10 pushed a commit to branch 4.3
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/4.3 by this push:
new 68f650ccde6 MINOR: Moving to atomic update of share sessions on initial epoch (#23510)
68f650ccde6 is described below
commit 68f650ccde6d574f0b7e4efb1bde9734e93eb99d
Author: Apoorv Mittal <ap...@gmail.com>
AuthorDate: Fri Sep 18 17:12:17 2026 +0100
MINOR: Moving to atomic update of share sessions on initial epoch (#23510)
ShareSessionCache `.remove` and `.maybeCreateSession` are each
individually synchronized — but as two separate calls, not one atomic
unit. If two connections sent an INITIAL_EPOCH request for the same
(groupId, memberId), *faulty client*, at nearly the same time then both
proceed to create a session. `maybeCreateSession` increments
numMembersPerGroup and numPartitions every time it's called. The
sessions map only ever holds one entry per key (second write overwrites
the first), but the counters had already been bumped twice.
on disconnect, only the connection whose mapping still pointed at a live
session gets to decrement the counters. So numMembersPerGroup for that
group gets stuck above zero forever, which means onGroupEmpty never
fires.
Reviewers: Andrew Schofield <as...@confluent.io>, Manikumar Reddy <ma...@gmail.com>
---
.../kafka/server/share/SharePartitionManager.java | 20 ++++---
.../server/share/SharePartitionManagerTest.java | 66 +++++++++++++++++++++-
.../server/share/session/ShareSessionCache.java | 2 +-
3 files changed, 78 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/kafka/server/share/SharePartitionManager.java b/core/src/main/java/kafka/server/share/SharePartitionManager.java
index bb57c56caf3..5f39e6be7ed 100644
--- a/core/src/main/java/kafka/server/share/SharePartitionManager.java
+++ b/core/src/main/java/kafka/server/share/SharePartitionManager.java
@@ -477,15 +477,19 @@ public class SharePartitionManager implements AutoCloseable {
log.error("Acknowledge data present in Initial Fetch Request for group {} member {}", groupId, memberId);
throw Errors.INVALID_REQUEST.exception();
}
- if (cache.remove(key) != null) {
- log.debug("Removed share session with key {}", key);
+ // The remove and the subsequent create must happen atomically under the cache's lock.
+ ShareSessionKey responseShareSessionKey;
+ synchronized (cache) {
+ if (cache.remove(key) != null) {
+ log.debug("Removed share session with key {}", key);
+ }
+ ImplicitLinkedHashCollection<CachedSharePartition> cachedSharePartitions = new
+ ImplicitLinkedHashCollection<>(shareFetchData.size());
+ shareFetchData.forEach(topicIdPartition ->
+ cachedSharePartitions.mustAdd(new CachedSharePartition(topicIdPartition, false)));
+ responseShareSessionKey = cache.maybeCreateSession(groupId, memberId,
+ cachedSharePartitions, clientConnectionId);
}
- ImplicitLinkedHashCollection<CachedSharePartition> cachedSharePartitions = new
- ImplicitLinkedHashCollection<>(shareFetchData.size());
- shareFetchData.forEach(topicIdPartition ->
- cachedSharePartitions.mustAdd(new CachedSharePartition(topicIdPartition, false)));
- ShareSessionKey responseShareSessionKey = cache.maybeCreateSession(groupId, memberId,
- cachedSharePartitions, clientConnectionId);
if (responseShareSessionKey == null) {
log.error("Could not create a share session for group {} member {}", groupId, memberId);
throw Errors.SHARE_SESSION_LIMIT_REACHED.exception();
diff --git a/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java b/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
index 0cb2f47fc4f..d7ed198104c 100644
--- a/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
+++ b/core/src/test/java/kafka/server/share/SharePartitionManagerTest.java
@@ -109,7 +109,12 @@ import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import java.util.stream.IntStream;
import scala.Tuple2;
import scala.collection.Seq;
@@ -343,10 +348,69 @@ public class SharePartitionManagerTest {
// and delete the older one.
ShareFetchContext context2 = sharePartitionManager.newContext(groupId, reqData, EMPTY_PART_LIST, memberId, ShareRequestMetadata.INITIAL_EPOCH, false, CONNECTION_ID);
assertInstanceOf(ShareSessionContext.class, context2);
- assertFalse(((ShareSessionContext) context1).isSubsequent());
+ assertFalse(((ShareSessionContext) context2).isSubsequent());
assertEquals(1, cache.size());
}
+ @Test
+ @SuppressWarnings("resource")
+ public void testConcurrentInitialEpochRequestsClearsState() throws Exception {
+ // Enough iterations to make catching a regression here reliable, tests complete under a sec
+ // with 1000 iterations.
+ int iterations = 1000;
+ ShareSessionCache cache = new ShareSessionCache(iterations * 2);
+ SharePartitionCache partitionCache = new SharePartitionCache();
+ sharePartitionManager = SharePartitionManagerBuilder.builder()
+ .withCache(cache)
+ .withPartitionCache(partitionCache)
+ .withReplicaManager(mockReplicaManager)
+ .build();
+
+ SharePartition sharePartition = mock(SharePartition.class);
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < iterations; i++) {
+ String groupId = "grp-" + i;
+ String memberId = Uuid.randomUuid().toString();
+ String connectionA = "conn-a-" + i;
+ String connectionB = "conn-b-" + i;
+ TopicIdPartition tp = new TopicIdPartition(Uuid.randomUuid(),
+ new TopicPartition("topic-" + i, 0));
+ List<TopicIdPartition> reqData = List.of(tp);
+
+ partitionCache.computeIfAbsent(new SharePartitionKey(groupId, tp),
+ k -> sharePartition);
+
+ Future<?> futureA = executor.submit(
+ () -> sharePartitionManager.newContext(groupId, reqData,
+ EMPTY_PART_LIST, memberId, ShareRequestMetadata.INITIAL_EPOCH, false,
+ connectionA));
+ Future<?> futureB = executor.submit(
+ () -> sharePartitionManager.newContext(groupId, reqData,
+ EMPTY_PART_LIST, memberId, ShareRequestMetadata.INITIAL_EPOCH, false,
+ connectionB));
+ futureA.get(10, TimeUnit.SECONDS);
+ futureB.get(10, TimeUnit.SECONDS);
+
+ cache.connectionDisconnectListener().onDisconnect(connectionA);
+ cache.connectionDisconnectListener().onDisconnect(connectionB);
+ }
+ } finally {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+ }
+
+ // Every session must be fully cleaned up. No leaked sessions, no phantom member/partition counts,
+ // and no share partitions retained past the point where the group should have gone empty.
+ assertEquals(0, cache.size());
+ assertEquals(0, cache.totalPartitions());
+ assertEquals(0, partitionCache.size());
+ IntStream.range(0, iterations).forEach(i -> {
+ Integer numMembers = cache.numMembers("grp-" + i);
+ assertEquals(0, numMembers == null ? 0 : numMembers);
+ });
+ }
+
@Test
public void testNewContext() {
ShareSessionCache cache = new ShareSessionCache(10);
diff --git a/server/src/main/java/org/apache/kafka/server/share/session/ShareSessionCache.java b/server/src/main/java/org/apache/kafka/server/share/session/ShareSessionCache.java
index 01051698039..5966251b290 100644
--- a/server/src/main/java/org/apache/kafka/server/share/session/ShareSessionCache.java
+++ b/server/src/main/java/org/apache/kafka/server/share/session/ShareSessionCache.java
@@ -266,7 +266,7 @@ public class ShareSessionCache {
}
// Visible for testing.
- Integer numMembers(String groupId) {
+ public Integer numMembers(String groupId) {
return numMembersPerGroup.get(groupId);
}