You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2024/01/22 09:16:27 UTC
(kafka) branch trunk updated: KAFKA-16147; Partition is assigned to two members at the same time (#15212)
This is an automated email from the ASF dual-hosted git repository.
dajac pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new cf90382fb92 KAFKA-16147; Partition is assigned to two members at the same time (#15212)
cf90382fb92 is described below
commit cf90382fb92374f526a2b18fa124e2f5e2ed5249
Author: David Jacot <dj...@confluent.io>
AuthorDate: Mon Jan 22 01:16:20 2024 -0800
KAFKA-16147; Partition is assigned to two members at the same time (#15212)
We had a case where a partition got assigned to two members and we found a bug in the partition epochs bookkeeping. Basically, when a member has a partition pending revocation re-assigned to him before the revocation is completed, the partition epoch is lost. Here is an example of such transition:
```
[2024-01-16 12:10:52,613] INFO [GroupCoordinator id=1 topic=__consumer_offsets partition=7] [GroupId rdkafkatest_rnd53b4eb0c2de343_0113u] Member M2 transitioned from CurrentAssignment(memberEpoch=11, previousMemberEpoch=9, targetMemberEpoch=14, state=revoking, assignedPartitions={}, partitionsPendingRevocation={EnZMikZURKiUoxZf0rozaA=[0, 1, 2, 3, 4, 5, 6, 7]}, partitionsPendingAssignment={IKXGrFR1Rv-Qes7Ummas6A=[0, 5]}) to CurrentAssignment(memberEpoch=15, previousMemberEpoch=11, targ [...]
```
This patch fixes the bug and also strengthen the partition epochs bookkeeping to not accept such invalid transitions.
Reviewers: Justine Olshan <jo...@confluent.io>
---
.../coordinator/group/consumer/ConsumerGroup.java | 52 ++++++----
.../group/consumer/ConsumerGroupTest.java | 106 +++++++++++++++++++++
2 files changed, 137 insertions(+), 21 deletions(-)
diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroup.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroup.java
index 216b06ac253..3b9390dd974 100644
--- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroup.java
+++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroup.java
@@ -861,19 +861,9 @@ public class ConsumerGroup implements Group {
ConsumerGroupMember oldMember,
ConsumerGroupMember newMember
) {
- if (oldMember == null) {
- addPartitionEpochs(newMember.assignedPartitions(), newMember.memberEpoch());
- addPartitionEpochs(newMember.partitionsPendingRevocation(), newMember.memberEpoch());
- } else {
- if (!oldMember.assignedPartitions().equals(newMember.assignedPartitions())) {
- removePartitionEpochs(oldMember.assignedPartitions());
- addPartitionEpochs(newMember.assignedPartitions(), newMember.memberEpoch());
- }
- if (!oldMember.partitionsPendingRevocation().equals(newMember.partitionsPendingRevocation())) {
- removePartitionEpochs(oldMember.partitionsPendingRevocation());
- addPartitionEpochs(newMember.partitionsPendingRevocation(), newMember.memberEpoch());
- }
- }
+ maybeRemovePartitionEpoch(oldMember);
+ addPartitionEpochs(newMember.assignedPartitions(), newMember.memberEpoch());
+ addPartitionEpochs(newMember.partitionsPendingRevocation(), newMember.memberEpoch());
}
/**
@@ -885,8 +875,8 @@ public class ConsumerGroup implements Group {
ConsumerGroupMember oldMember
) {
if (oldMember != null) {
- removePartitionEpochs(oldMember.assignedPartitions());
- removePartitionEpochs(oldMember.partitionsPendingRevocation());
+ removePartitionEpochs(oldMember.assignedPartitions(), oldMember.memberEpoch());
+ removePartitionEpochs(oldMember.partitionsPendingRevocation(), oldMember.memberEpoch());
}
}
@@ -894,21 +884,34 @@ public class ConsumerGroup implements Group {
* Removes the partition epochs based on the provided assignment.
*
* @param assignment The assignment.
+ * @param expectedEpoch The expected epoch.
+ * @throws IllegalStateException if the epoch does not match the expected one.
+ * package-private for testing.
*/
- private void removePartitionEpochs(
- Map<Uuid, Set<Integer>> assignment
+ void removePartitionEpochs(
+ Map<Uuid, Set<Integer>> assignment,
+ int expectedEpoch
) {
assignment.forEach((topicId, assignedPartitions) -> {
currentPartitionEpoch.compute(topicId, (__, partitionsOrNull) -> {
if (partitionsOrNull != null) {
- assignedPartitions.forEach(partitionsOrNull::remove);
+ assignedPartitions.forEach(partitionId -> {
+ Integer prevValue = partitionsOrNull.remove(partitionId);
+ if (prevValue != expectedEpoch) {
+ throw new IllegalStateException(
+ String.format("Cannot remove the epoch %d from %s-%s because the partition is " +
+ "still owned at a different epoch %d", expectedEpoch, topicId, partitionId, prevValue));
+ }
+ });
if (partitionsOrNull.isEmpty()) {
return null;
} else {
return partitionsOrNull;
}
} else {
- return null;
+ throw new IllegalStateException(
+ String.format("Cannot remove the epoch %d from %s because it does not have any epoch",
+ expectedEpoch, topicId));
}
});
});
@@ -919,8 +922,10 @@ public class ConsumerGroup implements Group {
*
* @param assignment The assignment.
* @param epoch The new epoch.
+ * @throws IllegalStateException if the partition already has an epoch assigned.
+ * package-private for testing.
*/
- private void addPartitionEpochs(
+ void addPartitionEpochs(
Map<Uuid, Set<Integer>> assignment,
int epoch
) {
@@ -930,7 +935,12 @@ public class ConsumerGroup implements Group {
partitionsOrNull = new TimelineHashMap<>(snapshotRegistry, assignedPartitions.size());
}
for (Integer partitionId : assignedPartitions) {
- partitionsOrNull.put(partitionId, epoch);
+ Integer prevValue = partitionsOrNull.put(partitionId, epoch);
+ if (prevValue != null) {
+ throw new IllegalStateException(
+ String.format("Cannot set the epoch of %s-%s to %d because the partition is " +
+ "still owned at epoch %d", topicId, partitionId, epoch, prevValue));
+ }
}
return partitionsOrNull;
});
diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupTest.java
index 8440a354d8f..7272e82f7f0 100644
--- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupTest.java
+++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupTest.java
@@ -215,6 +215,112 @@ public class ConsumerGroupTest {
assertEquals(-1, consumerGroup.currentPartitionEpoch(fooTopicId, 9));
}
+ @Test
+ public void testUpdatingMemberUpdatesPartitionEpochWhenPartitionIsReassignedBeforeBeingRevoked() {
+ Uuid fooTopicId = Uuid.randomUuid();
+
+ ConsumerGroup consumerGroup = createConsumerGroup("foo");
+ ConsumerGroupMember member;
+
+ member = new ConsumerGroupMember.Builder("member")
+ .setMemberEpoch(10)
+ .setAssignedPartitions(Collections.emptyMap())
+ .setPartitionsPendingRevocation(mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)))
+ .build();
+
+ consumerGroup.updateMember(member);
+
+ assertEquals(10, consumerGroup.currentPartitionEpoch(fooTopicId, 1));
+
+ member = new ConsumerGroupMember.Builder(member)
+ .setMemberEpoch(11)
+ .setAssignedPartitions(mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)))
+ .setPartitionsPendingRevocation(Collections.emptyMap())
+ .build();
+
+ consumerGroup.updateMember(member);
+
+ assertEquals(11, consumerGroup.currentPartitionEpoch(fooTopicId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesPartitionEpochWhenPartitionIsNotReleased() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ ConsumerGroup consumerGroup = createConsumerGroup("foo");
+
+ ConsumerGroupMember m1 = new ConsumerGroupMember.Builder("m1")
+ .setMemberEpoch(10)
+ .setAssignedPartitions(mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)))
+ .build();
+
+ consumerGroup.updateMember(m1);
+
+ ConsumerGroupMember m2 = new ConsumerGroupMember.Builder("m2")
+ .setMemberEpoch(10)
+ .setAssignedPartitions(mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)))
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> consumerGroup.updateMember(m2));
+ }
+
+ @Test
+ public void testRemovePartitionEpochs() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ ConsumerGroup consumerGroup = createConsumerGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> consumerGroup.removePartitionEpochs(
+ mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)
+ ),
+ 10
+ ));
+
+ ConsumerGroupMember m1 = new ConsumerGroupMember.Builder("m1")
+ .setMemberEpoch(10)
+ .setAssignedPartitions(mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)))
+ .build();
+
+ consumerGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> consumerGroup.removePartitionEpochs(
+ mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)
+ ),
+ 11
+ ));
+ }
+
+ @Test
+ public void testAddPartitionEpochs() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ ConsumerGroup consumerGroup = createConsumerGroup("foo");
+
+ consumerGroup.addPartitionEpochs(
+ mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)
+ ),
+ 10
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> consumerGroup.addPartitionEpochs(
+ mkAssignment(
+ mkTopicAssignment(fooTopicId, 1)
+ ),
+ 11
+ ));
+ }
+
@Test
public void testDeletingMemberRemovesPartitionEpoch() {
Uuid fooTopicId = Uuid.randomUuid();