You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by sq...@apache.org on 2026/09/24 09:44:25 UTC
(kafka) branch trunk updated: KAFKA-20292 [6/N]: Add TargetAssignmentRecordsBuilder (#22487)
This is an automated email from the ASF dual-hosted git repository.
squah-confluent pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6664fb9e97f KAFKA-20292 [6/N]: Add TargetAssignmentRecordsBuilder (#22487)
6664fb9e97f is described below
commit 6664fb9e97ff65ade2519ea68725e2e65b49eb2c
Author: Sean Quah <sq...@confluent.io>
AuthorDate: Thu Sep 24 10:44:19 2026 +0100
KAFKA-20292 [6/N]: Add TargetAssignmentRecordsBuilder (#22487)
To accommodate asynchronous assignors, such as assignors offloaded to
the background thread pool and client-side assignors, we require a
builder to turn a new target assignment into records to be persisted.
Previously, the logic lived within the TargetAssignmentBuilder classes.
We copy the logic out into its own builder class, keeping the existing
logic within the TargetAssignmentBuilders for now.
Future commits will remove the record-building logic from the
TargetAssignmentBuilder classes and update callers to use the new
TargetAssignmentRecordsBuilders.
Additionally, we have to handle concurrent member operations such as
leaves and static member replacements when building the target
assignment records for asynchronous assignors. These member operations
update the target assignment directly. Once the new target assignment is
ready, we have to apply any member operations we missed to it. That is,
we act as if the member removals and static member replacements are
reordered after the assignment calculation.
Reviewers: David Jacot <da...@gmail.com>, David Jacot
<da...@apache.org>
---
.../group/TargetAssignmentRecordsBuilder.java | 438 +++++++++++++++++
.../group/TargetAssignmentRecordsBuilderTest.java | 532 +++++++++++++++++++++
2 files changed, 970 insertions(+)
diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java
new file mode 100644
index 00000000000..6dfada98169
--- /dev/null
+++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group;
+
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.modern.Assignment;
+import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers;
+import org.apache.kafka.coordinator.group.streams.TasksTuple;
+
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Builds the records to persist a new target assignment for a group.
+ *
+ * Records are only created for members which have a new target assignment. If
+ * their assignment did not change, no new record is needed.
+ *
+ * When a member is deleted, it is assumed that its target assignment record
+ * is deleted as part of the member deletion process. In other words, this class
+ * does not yield a tombstone for removed members.
+ *
+ * @param <A> The member's target assignment type.
+ */
+public abstract class TargetAssignmentRecordsBuilder<A> {
+
+ /**
+ * The logger.
+ */
+ private final Logger log;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The target assignment metadata.
+ */
+ private TargetAssignmentMetadata targetAssignmentMetadata;
+
+ /**
+ * The static members in the group at the time the new target assignment was computed.
+ */
+ private Map<String, String> previousStaticMembers;
+
+ /**
+ * The current member ids in the group.
+ */
+ private Set<String> currentMemberIds;
+
+ /**
+ * The current static members in the group.
+ */
+ private Map<String, String> currentStaticMembers;
+
+ /**
+ * The current target assignment.
+ */
+ private Map<String, A> currentTargetAssignment;
+
+ /**
+ * The new target assignment.
+ */
+ private Map<String, A> newTargetAssignment;
+
+ /**
+ * Constructs the object.
+ *
+ * @param log The logger.
+ * @param groupId The group id.
+ */
+ public TargetAssignmentRecordsBuilder(
+ Logger log,
+ String groupId
+ ) {
+ this.log = log;
+ this.groupId = Objects.requireNonNull(groupId);
+ }
+
+ /**
+ * Sets the target assignment metadata.
+ *
+ * @param targetAssignmentMetadata The target assignment metadata.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) {
+ this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata);
+ return this;
+ }
+
+ /**
+ * Sets the static members in the group at the time the new target assignment was computed.
+ *
+ * @param previousStaticMembers The static members in the group at the time the new target assignment was computed.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) {
+ this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers);
+ return this;
+ }
+
+ /**
+ * Sets the current member ids in the group.
+ *
+ * @param currentMemberIds The current member ids in the group.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) {
+ this.currentMemberIds = Objects.requireNonNull(currentMemberIds);
+ return this;
+ }
+
+ /**
+ * Sets the current static members in the group.
+ *
+ * @param currentStaticMembers The current static members in the group.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) {
+ this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers);
+ return this;
+ }
+
+ /**
+ * Sets the current target assignment.
+ *
+ * @param currentTargetAssignment The current target assignment.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) {
+ this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment);
+ return this;
+ }
+
+ /**
+ * Sets the new target assignment.
+ *
+ * @param newTargetAssignment The new target assignment.
+ * @return This object.
+ */
+ public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) {
+ this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment);
+ return this;
+ }
+
+ /**
+ * Builds the records for the new target assignment.
+ *
+ * @return The records for the new target assignment.
+ */
+ public List<CoordinatorRecord> build() {
+ List<CoordinatorRecord> records = new ArrayList<>();
+ build(records);
+ return records;
+ }
+
+ /**
+ * Builds the records for the new target assignment.
+ *
+ * @param records The list to accumulate records.
+ */
+ @SuppressWarnings({"CyclomaticComplexity", "NPathComplexity"})
+ public void build(List<CoordinatorRecord> records) {
+ if (targetAssignmentMetadata == null)
+ throw new IllegalArgumentException("Target assignment metadata must be set.");
+ if (previousStaticMembers == null)
+ throw new IllegalArgumentException("Previous static members must be set.");
+ if (currentMemberIds == null)
+ throw new IllegalArgumentException("Current member ids must be set.");
+ if (currentStaticMembers == null)
+ throw new IllegalArgumentException("Current static members must be set.");
+ if (currentTargetAssignment == null)
+ throw new IllegalArgumentException("Current target assignment must be set.");
+ if (newTargetAssignment == null)
+ throw new IllegalArgumentException("New target assignment must be set.");
+
+ // The members in the group may have changed while the target assignment was computed.
+ // We want to act as if concurrent member operations such as leaves and static member
+ // replacements happened *after* the target assignment was computed.
+ //
+ // * When members leave the group, the group coordinator tombstones their target
+ // assignment.
+ // * When static members are replaced, the group coordinator moves the assignment from the
+ // old member id to the new member id.
+ //
+ // Thus, when building the new target assignment records,
+ // * we should not emit records for members that have left the group
+ // * we should relabel records with the latest member id for static members
+ //
+ // It is allowed by the protocol, but extremely unlikely, for static members to fully leave
+ // the group and rejoin with the same member id but different instance ids. We choose to
+ // match up assignments using member ids first, then fall back to instance ids.
+
+ // Build map of replacement member ids for static members that have churned.
+ Map<String, String> staticMemberIdRemapping = new HashMap<>();
+ for (Map.Entry<String, String> entry : previousStaticMembers.entrySet()) {
+ String instanceId = entry.getKey();
+ String oldMemberId = entry.getValue();
+ String newMemberId = currentStaticMembers.get(instanceId);
+
+ if (currentMemberIds.contains(oldMemberId)) {
+ // The old member id is still in the group. We must not create a remapping entry,
+ // otherwise we could give the same assignment to two different members.
+ continue;
+ }
+
+ if (newMemberId != null) {
+ if (newTargetAssignment.containsKey(newMemberId)) {
+ // The new member id has been in the group since before the assignment was
+ // computed. We want to prioritize matching assignments up by member id, so
+ // avoid creating a remapping entry. Note that we can't detect this if the
+ // assignor omits an entry for the member but nothing bad happens in that case.
+ continue;
+ }
+
+ log.debug("[GroupId {}] Previous static member {} with instance id {} has been replaced by {}, transferring target assignment.",
+ groupId, oldMemberId, instanceId, newMemberId);
+
+ staticMemberIdRemapping.put(newMemberId, oldMemberId);
+ } else {
+ log.debug("[GroupId {}] Previous static member {} with instance id {} has no replacement, discarding their target assignment.",
+ groupId, oldMemberId, instanceId);
+ }
+ }
+
+ if (log.isDebugEnabled()) {
+ for (Map.Entry<String, String> entry : currentStaticMembers.entrySet()) {
+ String instanceId = entry.getKey();
+ String newMemberId = entry.getValue();
+
+ if (newTargetAssignment.containsKey(newMemberId)) {
+ // The member id has been in the group the whole time.
+ continue;
+ }
+
+ if (!previousStaticMembers.containsKey(instanceId)) {
+ log.debug("[GroupId {}] Current static member {} with instance id {} has no previous static member and will receive an empty target assignment.",
+ groupId, newMemberId, instanceId);
+ }
+ }
+
+ for (String memberId : newTargetAssignment.keySet()) {
+ if (currentMemberIds.contains(memberId)) {
+ // The member id has been in the group the whole time.
+ continue;
+ }
+
+ log.debug("[GroupId {}] Member {} has left the group, discarding their target assignment unless they were static and a corresponding static member exists.",
+ groupId, memberId);
+ }
+ }
+
+ // Generate target assignment records for all current members.
+ // We must take care to generate empty assignment records for members without an entry in
+ // newTargetAssignment, otherwise we may leave partitions double-assigned if a custom
+ // assignor omits entries for members when it decides to give them an empty assignment.
+ for (String memberId : currentMemberIds) {
+ String previousMemberId = staticMemberIdRemapping.getOrDefault(memberId, memberId);
+
+ A oldMemberAssignment = currentTargetAssignment.get(memberId);
+ A newMemberAssignment = newTargetAssignment.get(previousMemberId);
+ if (newMemberAssignment == null) {
+ if (!staticMemberIdRemapping.containsKey(memberId)) {
+ log.debug("[GroupId {}] Member {} is new or has no target assignment. The member will receive an empty target assignment.",
+ groupId, memberId);
+ }
+
+ newMemberAssignment = emptyMemberAssignment();
+ }
+
+ if (oldMemberAssignment == null || !newMemberAssignment.equals(oldMemberAssignment)) {
+ // If the member had no assignment or had a different assignment, we
+ // create a record for the new assignment.
+ records.add(newTargetAssignmentRecord(
+ groupId,
+ memberId,
+ newMemberAssignment
+ ));
+ }
+ }
+
+ // Bump the target assignment epoch.
+ records.add(newTargetAssignmentMetadataRecord(
+ groupId,
+ targetAssignmentMetadata.assignmentEpoch(),
+ targetAssignmentMetadata.assignmentTimestamp()
+ ));
+ }
+
+ protected abstract A emptyMemberAssignment();
+
+ protected abstract CoordinatorRecord newTargetAssignmentRecord(
+ String groupId,
+ String memberId,
+ A memberAssignment
+ );
+
+ protected abstract CoordinatorRecord newTargetAssignmentMetadataRecord(
+ String groupId,
+ int assignmentEpoch,
+ long assignmentTimestampMs
+ );
+
+ public static class ConsumerTargetAssignmentRecordsBuilder extends TargetAssignmentRecordsBuilder<Assignment> {
+
+ public ConsumerTargetAssignmentRecordsBuilder(Logger log, String groupId) {
+ super(log, groupId);
+ }
+
+ @Override
+ protected Assignment emptyMemberAssignment() {
+ return Assignment.EMPTY;
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentRecord(
+ String groupId,
+ String memberId,
+ Assignment memberAssignment
+ ) {
+ return GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord(
+ groupId,
+ memberId,
+ memberAssignment.partitions()
+ );
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentMetadataRecord(
+ String groupId,
+ int assignmentEpoch,
+ long assignmentTimestampMs
+ ) {
+ return GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord(
+ groupId,
+ assignmentEpoch,
+ assignmentTimestampMs
+ );
+ }
+ }
+
+ public static class ShareTargetAssignmentRecordsBuilder extends TargetAssignmentRecordsBuilder<Assignment> {
+
+ public ShareTargetAssignmentRecordsBuilder(Logger log, String groupId) {
+ super(log, groupId);
+ }
+
+ @Override
+ protected Assignment emptyMemberAssignment() {
+ return Assignment.EMPTY;
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentRecord(
+ String groupId,
+ String memberId,
+ Assignment memberAssignment
+ ) {
+ return GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentRecord(
+ groupId,
+ memberId,
+ memberAssignment.partitions()
+ );
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentMetadataRecord(
+ String groupId,
+ int assignmentEpoch,
+ long assignmentTimestampMs
+ ) {
+ return GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentMetadataRecord(
+ groupId,
+ assignmentEpoch,
+ assignmentTimestampMs
+ );
+ }
+ }
+
+ public static class StreamsTargetAssignmentRecordsBuilder extends TargetAssignmentRecordsBuilder<TasksTuple> {
+
+ public StreamsTargetAssignmentRecordsBuilder(Logger log, String groupId) {
+ super(log, groupId);
+ }
+
+ @Override
+ protected TasksTuple emptyMemberAssignment() {
+ return TasksTuple.EMPTY;
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentRecord(
+ String groupId,
+ String memberId,
+ TasksTuple memberAssignment
+ ) {
+ return StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord(
+ groupId,
+ memberId,
+ memberAssignment
+ );
+ }
+
+ @Override
+ protected CoordinatorRecord newTargetAssignmentMetadataRecord(
+ String groupId,
+ int assignmentEpoch,
+ long assignmentTimestampMs
+ ) {
+ return StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord(
+ groupId,
+ assignmentEpoch,
+ assignmentTimestampMs
+ );
+ }
+ }
+}
diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilderTest.java
new file mode 100644
index 00000000000..c64411adba5
--- /dev/null
+++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilderTest.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.internals.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.modern.Assignment;
+import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers;
+import org.apache.kafka.coordinator.group.streams.TaskRole;
+import org.apache.kafka.coordinator.group.streams.TasksTuple;
+
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals;
+import static org.apache.kafka.coordinator.group.AssignmentTestUtil.mkAssignment;
+import static org.apache.kafka.coordinator.group.AssignmentTestUtil.mkTopicAssignment;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TargetAssignmentRecordsBuilderTest {
+
+ private static final Logger LOG = new LogContext().logger(TargetAssignmentRecordsBuilder.class);
+
+ @Test
+ public void testEmpty() {
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of())
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of())
+ .withNewTargetAssignment(Map.of())
+ .build();
+
+ assertEquals(List.of(
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testAssignmentHasNotChanged() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ Uuid barTopicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1", "member-2"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1, 2),
+ mkTopicAssignment(barTopicId, 0, 1, 2)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 3, 4, 5),
+ mkTopicAssignment(barTopicId, 3, 4, 5)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1, 2),
+ mkTopicAssignment(barTopicId, 0, 1, 2)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 3, 4, 5),
+ mkTopicAssignment(barTopicId, 3, 4, 5)
+ ))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testAssignmentSwapped() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ Uuid barTopicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1", "member-2"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1, 2),
+ mkTopicAssignment(barTopicId, 0, 1, 2)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 3, 4, 5),
+ mkTopicAssignment(barTopicId, 3, 4, 5)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 3, 4, 5),
+ mkTopicAssignment(barTopicId, 3, 4, 5)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1, 2),
+ mkTopicAssignment(barTopicId, 0, 1, 2)
+ ))
+ ))
+ .build();
+
+ assertUnorderedRecordsEquals(List.of(
+ List.of(
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment(
+ mkTopicAssignment(fooTopicId, 3, 4, 5),
+ mkTopicAssignment(barTopicId, 3, 4, 5)
+ )),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-2", mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1, 2),
+ mkTopicAssignment(barTopicId, 0, 1, 2)
+ ))
+ ),
+ List.of(GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L))
+ ), records);
+ }
+
+ @Test
+ public void testPartialAssignmentUpdate() {
+ Uuid fooTopicId = Uuid.randomUuid();
+ Uuid barTopicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1", "member-2", "member-3"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1),
+ mkTopicAssignment(barTopicId, 0, 1)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 2, 3),
+ mkTopicAssignment(barTopicId, 2, 3)
+ )),
+ "member-3", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 4, 5),
+ mkTopicAssignment(barTopicId, 4, 5)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 0, 1),
+ mkTopicAssignment(barTopicId, 0, 1)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 2, 3, 4),
+ mkTopicAssignment(barTopicId, 2, 3, 4)
+ )),
+ "member-3", new Assignment(mkAssignment(
+ mkTopicAssignment(fooTopicId, 5),
+ mkTopicAssignment(barTopicId, 5)
+ ))
+ ))
+ .build();
+
+ assertUnorderedRecordsEquals(List.of(
+ List.of(
+ // Member 1 has no record because its assignment did not change.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-2", mkAssignment(
+ mkTopicAssignment(fooTopicId, 2, 3, 4),
+ mkTopicAssignment(barTopicId, 2, 3, 4)
+ )),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-3", mkAssignment(
+ mkTopicAssignment(fooTopicId, 5),
+ mkTopicAssignment(barTopicId, 5)
+ ))
+ ),
+ List.of(GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L))
+ ), records);
+ }
+
+ @Test
+ public void testAssignmentEmptyWhenOmitted() {
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of())
+ // The assignor did not include an entry for member 1.
+ .withNewTargetAssignment(Map.of())
+ .build();
+
+ assertEquals(List.of(
+ // An empty target assignment is written for member even though the assignor did not include an entry for it.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment()),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testMemberLeft() {
+ Uuid topicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ // Member 2 has left.
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2, 3)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 2, 3)
+ ))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment(
+ mkTopicAssignment(topicId, 0, 1)
+ )),
+ // No record is written for member 2.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testMemberJoined() {
+ // This test is equivalent to testAssignmentEmptyWhenOmitted.
+
+ Uuid topicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ // Member 2 has joined.
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1", "member-2"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2)
+ ))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ // An empty target assignment is written for member 2.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-2", mkAssignment()),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testStaticMemberReplaced() {
+ Uuid topicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ // Static member 2 has been replaced with member 3.
+ .withPreviousStaticMembers(Map.of("instance-id", "member-2"))
+ .withCurrentMemberIds(Set.of("member-1", "member-3"))
+ .withCurrentStaticMembers(Map.of("instance-id", "member-3"))
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2)
+ )),
+ "member-3", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 3, 4, 5)
+ ))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2)
+ )),
+ "member-2", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId, 3, 4, 5, 6)
+ ))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ // Member 2's assignment is given to member 3.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-3", mkAssignment(
+ mkTopicAssignment(topicId, 3, 4, 5, 6)
+ )),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testStaticMemberAndMemberIdConflict() {
+ // This scenario can arise when static members leave the group fully and rejoin with the
+ // same member id but different instance id. It will never happen when using the official
+ // Java client.
+ Uuid topicId1 = Uuid.randomUuid();
+ Uuid topicId2 = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ // instance-id-1 has "moved" from member id 1 to member id 2.
+ // member id 1 has been around the whole time and member id 2 is new.
+ // instance-id-2 has "moved" from member id 3 to member id 4.
+ // member id 3 left and member id 4 has been around the whole time.
+ .withPreviousStaticMembers(Map.of("instance-id-1", "member-1", "instance-id-2", "member-3"))
+ .withCurrentMemberIds(Set.of("member-1", "member-2", "member-4"))
+ .withCurrentStaticMembers(Map.of("instance-id-1", "member-2", "instance-id-2", "member-4"))
+ .withCurrentTargetAssignment(Map.of())
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId1, 0, 1, 2, 3)
+ )),
+ "member-3", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId2, 0, 1, 2, 3)
+ )),
+ "member-4", new Assignment(mkAssignment(
+ mkTopicAssignment(topicId2, 4, 5, 6, 7)
+ ))
+ ))
+ .build();
+
+ assertUnorderedRecordsEquals(List.of(
+ List.of(
+ // Member 1 gets the target assignment because we prioritize matching on member id.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment(
+ mkTopicAssignment(topicId1, 0, 1, 2, 3)
+ )),
+ // Member 2 does not get member 1's target assignment even though it has a matching instance id.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-2", mkAssignment()),
+ // Member 3 gets nothing because it is no longer in the group.
+ // Member 4 gets the target assignment of member id 4, even though it has member 3's instance id,
+ // because we prioritize matching on member id.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-4", mkAssignment(
+ mkTopicAssignment(topicId2, 4, 5, 6, 7)
+ ))
+ ),
+ List.of(GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L))
+ ), records);
+ }
+
+ @Nested
+ public class ConsumerTargetAssignmentRecordsBuilderTest {
+
+ @Test
+ public void testEmptyMemberAssignment() {
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of())
+ .withNewTargetAssignment(Map.of())
+ .build();
+
+ assertEquals(List.of(
+ // An empty target assignment is written for member even though the assignor did not include an entry for it.
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment()),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testRecords() {
+ Uuid topicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ConsumerTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(mkTopicAssignment(topicId, 0)))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(mkTopicAssignment(topicId, 0, 1, 2, 3)))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2, 3)
+ )),
+ GroupCoordinatorRecordHelpers.newConsumerGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+ }
+
+ @Nested
+ public class ShareTargetAssignmentRecordsBuilderTest {
+
+ @Test
+ public void testEmptyMemberAssignment() {
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ShareTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of())
+ .withNewTargetAssignment(Map.of())
+ .build();
+
+ assertEquals(List.of(
+ // An empty target assignment is written for member even though the assignor did not include an entry for it.
+ GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment()),
+ GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testRecords() {
+ Uuid topicId = Uuid.randomUuid();
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.ShareTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(mkTopicAssignment(topicId, 0, 1)))
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", new Assignment(mkAssignment(mkTopicAssignment(topicId, 0, 1, 2, 3)))
+ ))
+ .build();
+
+ assertEquals(List.of(
+ GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentRecord("my-group", "member-1", mkAssignment(
+ mkTopicAssignment(topicId, 0, 1, 2, 3)
+ )),
+ GroupCoordinatorRecordHelpers.newShareGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+ }
+
+ @Nested
+ public class StreamsTargetAssignmentRecordsBuilderTest {
+
+ @Test
+ public void testEmptyMemberAssignment() {
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.StreamsTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of())
+ .withNewTargetAssignment(Map.of())
+ .build();
+
+ assertEquals(List.of(
+ // An empty target assignment is written for member even though the assignor did not include an entry for it.
+ StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord("my-group", "member-1", TasksTuple.EMPTY),
+ StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+
+ @Test
+ public void testRecords() {
+ String subtopology1 = "subtopology-1";
+
+ List<CoordinatorRecord> records =
+ new TargetAssignmentRecordsBuilder.StreamsTargetAssignmentRecordsBuilder(LOG, "my-group")
+ .withTargetAssignmentMetadata(new TargetAssignmentMetadata(20, 12345L))
+ .withPreviousStaticMembers(Map.of())
+ .withCurrentMemberIds(Set.of("member-1"))
+ .withCurrentStaticMembers(Map.of())
+ .withCurrentTargetAssignment(Map.of(
+ "member-1", mkTasksTuple(TaskRole.ACTIVE,
+ mkTasks(subtopology1, 0)
+ )
+ ))
+ .withNewTargetAssignment(Map.of(
+ "member-1", mkTasksTuple(TaskRole.ACTIVE,
+ mkTasks(subtopology1, 0, 1, 2, 3)
+ )
+ ))
+ .build();
+
+ assertEquals(List.of(
+ StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord("my-group", "member-1", mkTasksTuple(TaskRole.ACTIVE,
+ mkTasks(subtopology1, 0, 1, 2, 3)
+ )),
+ StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord("my-group", 20, 12345L)
+ ), records);
+ }
+ }
+}