You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2022/11/29 08:53:52 UTC
[kafka] branch 3.3 updated: MINOR: JoinGroup and SyncGroup callbacks should catch exceptions (#12910)
This is an automated email from the ASF dual-hosted git repository.
dajac pushed a commit to branch 3.3
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/3.3 by this push:
new 0de727af1e6 MINOR: JoinGroup and SyncGroup callbacks should catch exceptions (#12910)
0de727af1e6 is described below
commit 0de727af1e66149b55edb727191fe7f01e527c1b
Author: David Jacot <dj...@confluent.io>
AuthorDate: Tue Nov 29 09:48:31 2022 +0100
MINOR: JoinGroup and SyncGroup callbacks should catch exceptions (#12910)
We recently had a bug causing the JoinGroup callback to thrown an exception (https://github.com/apache/kafka/pull/12909). When it happens, the exception is propagated to the caller and the JoinGroup callback is never completed. To make it worst, the member whose callback failed become a zombie because the group coordinator does not expire member with a pending callback.
This patch catch exceptions for both invocation of JoinGroup and SyncGroup callbacks and retry to complete them with a `UNKNOWN_SERVER_ERROR` error if they failed.
Reviewers: Jason Gustafson <ja...@confluent.io>
---
.../kafka/coordinator/group/GroupMetadata.scala | 24 ++++++++---
.../coordinator/group/GroupMetadataTest.scala | 49 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 5 deletions(-)
diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
index c2e2176856c..b5b0b2df271 100644
--- a/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
+++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadata.scala
@@ -546,9 +546,16 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
def maybeInvokeJoinCallback(member: MemberMetadata,
joinGroupResult: JoinGroupResult): Unit = {
if (member.isAwaitingJoin) {
- member.awaitingJoinCallback(joinGroupResult)
- member.awaitingJoinCallback = null
- numMembersAwaitingJoin -= 1
+ try {
+ member.awaitingJoinCallback(joinGroupResult)
+ } catch {
+ case t: Throwable =>
+ error(s"Failed to invoke join callback for $member due to ${t.getMessage}.", t)
+ member.awaitingJoinCallback(JoinGroupResult(member.memberId, Errors.UNKNOWN_SERVER_ERROR))
+ } finally {
+ member.awaitingJoinCallback = null
+ numMembersAwaitingJoin -= 1
+ }
}
}
@@ -558,8 +565,15 @@ private[group] class GroupMetadata(val groupId: String, initialState: GroupState
def maybeInvokeSyncCallback(member: MemberMetadata,
syncGroupResult: SyncGroupResult): Boolean = {
if (member.isAwaitingSync) {
- member.awaitingSyncCallback(syncGroupResult)
- member.awaitingSyncCallback = null
+ try {
+ member.awaitingSyncCallback(syncGroupResult)
+ } catch {
+ case t: Throwable =>
+ error(s"Failed to invoke sync callback for $member due to ${t.getMessage}.", t)
+ member.awaitingSyncCallback(SyncGroupResult(Errors.UNKNOWN_SERVER_ERROR))
+ } finally {
+ member.awaitingSyncCallback = null
+ }
true
} else {
false
diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
index 98c0d93b88d..e750024c9fb 100644
--- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
+++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataTest.scala
@@ -620,6 +620,30 @@ class GroupMetadataTest {
assertFalse(member.isAwaitingJoin)
}
+ @Test
+ def testInvokeJoinCallbackFails(): Unit = {
+ val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
+ protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
+
+ var shouldFail = true
+ var result: Option[JoinGroupResult] = None
+ def joinCallback(joinGroupResult: JoinGroupResult): Unit = {
+ if (shouldFail) {
+ shouldFail = false
+ throw new Exception("Something went wrong!")
+ } else {
+ result = Some(joinGroupResult)
+ }
+ }
+
+ group.add(member, joinCallback)
+
+ group.maybeInvokeJoinCallback(member, JoinGroupResult(member.memberId, Errors.NONE))
+
+ assertEquals(Errors.UNKNOWN_SERVER_ERROR, result.get.error)
+ assertFalse(member.isAwaitingJoin)
+ }
+
@Test
def testNotInvokeJoinCallback(): Unit = {
val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
@@ -631,6 +655,31 @@ class GroupMetadataTest {
assertFalse(member.isAwaitingJoin)
}
+ @Test
+ def testInvokeSyncCallbackFails(): Unit = {
+ val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,
+ protocolType, List(("range", Array.empty[Byte]), ("roundrobin", Array.empty[Byte])))
+
+ var shouldFail = true
+ var result: Option[SyncGroupResult] = None
+ def syncCallback(syncGroupResult: SyncGroupResult): Unit = {
+ if (shouldFail) {
+ shouldFail = false
+ throw new Exception("Something went wrong!")
+ } else {
+ result = Some(syncGroupResult)
+ }
+ }
+
+ group.add(member)
+ member.awaitingSyncCallback = syncCallback
+
+ val invoked = group.maybeInvokeSyncCallback(member, SyncGroupResult(Errors.NONE))
+ assertTrue(invoked)
+ assertEquals(Errors.UNKNOWN_SERVER_ERROR, result.get.error)
+ assertFalse(member.isAwaitingSync)
+ }
+
@Test
def testInvokeSyncCallback(): Unit = {
val member = new MemberMetadata(memberId, None, clientId, clientHost, rebalanceTimeoutMs, sessionTimeoutMs,