You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2023/12/11 19:05:34 UTC
(kafka) branch trunk updated: KAFKA-15574; [2/N]: Client state machine updates (#14857)
This is an automated email from the ASF dual-hosted git repository.
dajac pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 839e75a7296 KAFKA-15574; [2/N]: Client state machine updates (#14857)
839e75a7296 is described below
commit 839e75a72968b697b05c6265219fc3bba583dd40
Author: Lianet Magrans <98...@users.noreply.github.com>
AuthorDate: Mon Dec 11 14:05:27 2023 -0500
KAFKA-15574; [2/N]: Client state machine updates (#14857)
This patch updates the client state machine, including:
* handling transitions to fatal and fenced while member is leaving the group.
* minor improvements addressing comments.
* support for static member leave group.
* minor fixes for handling assignments received.
Reviewers: David Jacot <dj...@confluent.io>
---
.../internals/HeartbeatRequestManager.java | 50 +--
.../clients/consumer/internals/MemberState.java | 8 +-
.../consumer/internals/MembershipManager.java | 10 +-
.../consumer/internals/MembershipManagerImpl.java | 173 +++++---
.../consumer/internals/AsyncKafkaConsumerTest.java | 4 +-
.../internals/HeartbeatRequestManagerTest.java | 9 +-
.../internals/MembershipManagerImplTest.java | 446 +++++++++++++++++----
7 files changed, 523 insertions(+), 177 deletions(-)
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
index 3632098a09b..3b099eb0766 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
@@ -22,7 +22,6 @@ import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollRes
import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent;
-import org.apache.kafka.common.TopicIdPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.errors.GroupAuthorizationException;
import org.apache.kafka.common.errors.RetriableException;
@@ -37,10 +36,9 @@ import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Collectors;
@@ -315,17 +313,15 @@ public class HeartbeatRequestManager implements RequestManager {
break;
case FENCED_MEMBER_EPOCH:
- message = String.format("GroupHeartbeatRequest failed because member ID %s with epoch %s is invalid. " +
- "Will abandon all partitions and rejoin the group",
+ message = String.format("GroupHeartbeatRequest failed for member %s because epoch %s is fenced.",
membershipManager.memberId(), membershipManager.memberEpoch());
logInfo(message, response, currentTimeMs);
membershipManager.transitionToFenced();
break;
case UNKNOWN_MEMBER_ID:
- message = String.format("GroupHeartbeatRequest failed because member of unknown ID %s with epoch %s is invalid. " +
- "Will abandon all partitions and rejoin the group",
- membershipManager.memberId(), membershipManager.memberEpoch());
+ message = String.format("GroupHeartbeatRequest failed because member %s is unknown.",
+ membershipManager.memberId());
logInfo(message, response, currentTimeMs);
membershipManager.transitionToFenced();
break;
@@ -485,17 +481,16 @@ public class HeartbeatRequestManager implements RequestManager {
// ClientAssignors - not supported yet
- // TopicPartitions - only sent if has changed since the last heartbeat
- // Note that TopicIdPartition.toString is being avoided here so that
- // the string consists of just the topic ID and the partition.
- // When an assignment is received, we might not yet know the topic name
- // and then it is learnt subsequently by a metadata update.
- TreeSet<String> assignedPartitions = membershipManager.currentAssignment().stream()
- .map(tp -> tp.topicId() + "-" + tp.partition())
- .collect(Collectors.toCollection(TreeSet::new));
+ // TopicPartitions - only sent if it has changed since the last heartbeat. Note that
+ // the string consists of just the topic ID and the partitions. When an assignment is
+ // received, we might not yet know the topic name, and then it is learnt subsequently
+ // by a metadata update.
+ TreeSet<String> assignedPartitions = membershipManager.currentAssignment().entrySet().stream()
+ .map(entry -> entry.getKey() + "-" + entry.getValue())
+ .collect(Collectors.toCollection(TreeSet::new));
if (!assignedPartitions.equals(sentFields.topicPartitions)) {
List<ConsumerGroupHeartbeatRequestData.TopicPartitions> topicPartitions =
- buildTopicPartitionsList(membershipManager.currentAssignment());
+ buildTopicPartitionsList(membershipManager.currentAssignment());
data.setTopicPartitions(topicPartitions);
sentFields.topicPartitions = assignedPartitions;
}
@@ -503,21 +498,12 @@ public class HeartbeatRequestManager implements RequestManager {
return data;
}
- private List<ConsumerGroupHeartbeatRequestData.TopicPartitions> buildTopicPartitionsList(Set<TopicIdPartition> topicIdPartitions) {
- List<ConsumerGroupHeartbeatRequestData.TopicPartitions> result = new ArrayList<>();
- Map<Uuid, List<Integer>> partitionsPerTopicId = new HashMap<>();
- for (TopicIdPartition topicIdPartition : topicIdPartitions) {
- Uuid topicId = topicIdPartition.topicId();
- partitionsPerTopicId.computeIfAbsent(topicId, __ -> new ArrayList<>()).add(topicIdPartition.partition());
- }
- for (Map.Entry<Uuid, List<Integer>> entry : partitionsPerTopicId.entrySet()) {
- Uuid topicId = entry.getKey();
- List<Integer> partitions = entry.getValue();
- result.add(new ConsumerGroupHeartbeatRequestData.TopicPartitions()
- .setTopicId(topicId)
- .setPartitions(partitions));
- }
- return result;
+ private List<ConsumerGroupHeartbeatRequestData.TopicPartitions> buildTopicPartitionsList(Map<Uuid, SortedSet<Integer>> topicIdPartitions) {
+ return topicIdPartitions.entrySet().stream().map(
+ entry -> new ConsumerGroupHeartbeatRequestData.TopicPartitions()
+ .setTopicId(entry.getKey())
+ .setPartitions(new ArrayList<>(entry.getValue())))
+ .collect(Collectors.toList());
}
// Fields of ConsumerHeartbeatRequest sent in the most recent request
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java
index 35d70b0fc99..088d1ccd346 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MemberState.java
@@ -104,15 +104,17 @@ public enum MemberState {
// Valid state transitions
static {
- STABLE.previousValidStates = Arrays.asList(JOINING, ACKNOWLEDGING);
+ STABLE.previousValidStates = Arrays.asList(JOINING, ACKNOWLEDGING, RECONCILING);
RECONCILING.previousValidStates = Arrays.asList(STABLE, JOINING, ACKNOWLEDGING);
ACKNOWLEDGING.previousValidStates = Arrays.asList(RECONCILING);
- FATAL.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING);
+ FATAL.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING,
+ PREPARE_LEAVING, LEAVING, UNSUBSCRIBED);
- FENCED.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING);
+ FENCED.previousValidStates = Arrays.asList(JOINING, STABLE, RECONCILING, ACKNOWLEDGING,
+ PREPARE_LEAVING, LEAVING);
JOINING.previousValidStates = Arrays.asList(FENCED, UNSUBSCRIBED);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java
index 4727daa0f64..b635eb3a0fd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManager.java
@@ -16,11 +16,12 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData;
+import java.util.Map;
import java.util.Optional;
-import java.util.Set;
+import java.util.SortedSet;
import java.util.concurrent.CompletableFuture;
/**
@@ -88,9 +89,10 @@ public interface MembershipManager {
Optional<String> serverAssignor();
/**
- * @return Current assignment for the member.
+ * @return Current assignment for the member as received from the broker (topic IDs and
+ * partitions). This is the last assignment that the member has successfully reconciled.
*/
- Set<TopicIdPartition> currentAssignment();
+ Map<Uuid, SortedSet<Integer>> currentAssignment();
/**
* Transition the member to the FENCED state, where the member will release the assignment by
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java
index 5f72dbf78d2..317126801ec 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImpl.java
@@ -39,15 +39,14 @@ import org.slf4j.Logger;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
/**
* Group manager for a single consumer that has a group id defined in the config
@@ -156,7 +155,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
/**
* Assignment that the member received from the server and successfully processed.
*/
- private Set<TopicIdPartition> currentAssignment;
+ private Map<Uuid, SortedSet<Integer>> currentAssignment;
/**
* Subscription state object holding the current assignment the member has for the topics it
@@ -198,7 +197,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
* received from the broker, even though they may not be ready to reconcile due to missing
* metadata.
*/
- private final Map<Uuid, List<Integer>> assignmentUnresolved;
+ private final Map<Uuid, SortedSet<Integer>> assignmentUnresolved;
/**
* Assignment received for which topic names have been resolved, so it's ready to be
@@ -264,7 +263,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
this.assignedTopicNamesCache = new HashMap<>();
this.assignmentUnresolved = new HashMap<>();
this.assignmentReadyToReconcile = new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR);
- this.currentAssignment = new HashSet<>();
+ this.currentAssignment = new HashMap<>();
this.log = logContext.logger(MembershipManagerImpl.class);
this.clientTelemetryReporter = clientTelemetryReporter;
}
@@ -280,7 +279,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
throw new IllegalStateException(String.format("Invalid state transition from %s to %s",
state, nextState));
}
- log.trace("Member {} transitioned from {} to {}.", memberId, state, nextState);
+ log.trace("Member {} with epoch {} transitioned from {} to {}.", memberId, memberEpoch, state, nextState);
this.state = nextState;
}
@@ -344,10 +343,29 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
ConsumerGroupHeartbeatResponseData.Assignment assignment = response.assignment();
if (assignment != null) {
- transitionTo(MemberState.RECONCILING);
replaceUnresolvedAssignmentWithNewAssignment(assignment);
- resolveMetadataForUnresolvedAssignment();
- reconcile();
+ if (!assignmentUnresolved.equals(currentAssignment)) {
+ // Transition the member to RECONCILING when receiving a new target
+ // assignment from the broker, different from the current assignment. Note that the
+ // reconciliation might not be triggered just yet because of missing metadata.
+ transitionTo(MemberState.RECONCILING);
+ assignmentReadyToReconcile.clear();
+ resolveMetadataForUnresolvedAssignment();
+ reconcile();
+ } else {
+ // Same assignment received, nothing to reconcile.
+ log.debug("Target assignment {} received from the broker is equals to the member " +
+ "current assignment {}. Nothing to reconcile.",
+ assignmentUnresolved, currentAssignment);
+ // Make sure we transition the member back to STABLE if it was RECONCILING (ex.
+ // member was RECONCILING unresolved assignments that were just removed by the
+ // broker).
+ if (state == MemberState.RECONCILING) {
+ // This is the case where a member was RECONCILING an unresolved
+ // assignment that was removed by the broker in a following assignment.
+ transitionTo(MemberState.STABLE);
+ }
+ }
} else if (allPendingAssignmentsReconciled()) {
transitionTo(MemberState.STABLE);
}
@@ -370,7 +388,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
ConsumerGroupHeartbeatResponseData.Assignment assignment) {
assignmentUnresolved.clear();
assignment.topicPartitions().forEach(topicPartitions ->
- assignmentUnresolved.put(topicPartitions.topicId(), topicPartitions.partitions()));
+ assignmentUnresolved.put(topicPartitions.topicId(), new TreeSet<>(topicPartitions.partitions())));
}
/**
@@ -378,6 +396,16 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
*/
@Override
public void transitionToFenced() {
+ if (state == MemberState.PREPARE_LEAVING || state == MemberState.LEAVING) {
+ log.debug("Member {} with epoch {} got fenced but it is already leaving the group " +
+ "with state {}, so it won't attempt to rejoin.", memberId, memberEpoch, state);
+ return;
+ }
+ if (state == MemberState.UNSUBSCRIBED) {
+ log.debug("Member {} with epoch {} got fenced but it already left the group, so it " +
+ "won't attempt to rejoin.", memberId, memberEpoch);
+ return;
+ }
transitionTo(MemberState.FENCED);
resetEpoch();
log.debug("Member {} with epoch {} transitioned to {} state. It will release its " +
@@ -390,7 +418,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
log.error("onPartitionsLost callback invocation failed while releasing assignment" +
" after member got fenced. Member will rejoin the group anyways.", error);
}
- updateSubscription(Collections.emptySet(), true);
+ updateSubscription(new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR), true);
transitionToJoining();
});
}
@@ -400,9 +428,16 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
*/
@Override
public void transitionToFatal() {
+ MemberState previousState = state;
transitionTo(MemberState.FATAL);
log.error("Member {} with epoch {} transitioned to {} state", memberId, memberEpoch, MemberState.FATAL);
+ if (previousState == MemberState.UNSUBSCRIBED) {
+ log.debug("Member {} with epoch {} got fatal error from the broker but it already " +
+ "left the group, so onPartitionsLost callback won't be triggered.", memberId, memberEpoch);
+ return;
+ }
+
// Release assignment
CompletableFuture<Void> callbackResult = invokeOnPartitionsLostCallback(subscriptions.assignedPartitions());
callbackResult.whenComplete((result, error) -> {
@@ -410,7 +445,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
log.error("onPartitionsLost callback invocation failed while releasing assignment" +
"after member failed with fatal error.", error);
}
- updateSubscription(Collections.emptySet(), true);
+ updateSubscription(new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR), true);
});
}
@@ -429,9 +464,12 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
* @param assignedPartitions Topic partitions to take as the new subscription assignment
* @param clearAssignments True if the pending assignments and metadata cache should be cleared
*/
- private void updateSubscription(Collection<TopicPartition> assignedPartitions,
+ private void updateSubscription(SortedSet<TopicIdPartition> assignedPartitions,
boolean clearAssignments) {
- subscriptions.assignFromSubscribed(assignedPartitions);
+ Collection<TopicPartition> assignedTopicPartitions = toTopicPartitionSet(assignedPartitions);
+ subscriptions.assignFromSubscribed(assignedTopicPartitions);
+ // Make assignment effective on the member group manager.
+ updateCurrentAssignment(assignedPartitions);
if (clearAssignments) {
clearPendingAssignmentsAndLocalNamesCache();
}
@@ -490,7 +528,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
CompletableFuture<Void> callbackResult = invokeOnPartitionsRevokedOrLostToReleaseAssignment();
callbackResult.whenComplete((result, error) -> {
// Clear the subscription, no matter if the callback execution failed or succeeded.
- updateSubscription(Collections.emptySet(), true);
+ updateSubscription(new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR), true);
// Transition to ensure that a heartbeat request is sent out to effectively leave the
// group (even in the case where the member had no assignment to release or when the
@@ -523,10 +561,10 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
CompletableFuture<Void> callbackResult;
if (droppedPartitions.isEmpty()) {
- // No assignment to release
+ // No assignment to release.
callbackResult = CompletableFuture.completedFuture(null);
} else {
- // Release assignment
+ // Release assignment.
if (memberEpoch > 0) {
// Member is part of the group. Invoke onPartitionsRevoked.
callbackResult = revokePartitions(droppedPartitions);
@@ -544,9 +582,16 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
* request is sent out with it.
*/
private void transitionToSendingLeaveGroup() {
- memberEpoch = ConsumerGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH;
+ if (state == MemberState.FATAL) {
+ log.warn("Member {} with epoch {} won't send leave group request because it is in " +
+ "FATAL state", memberId, memberEpoch);
+ return;
+ }
+ memberEpoch = groupInstanceId.isPresent() ?
+ ConsumerGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH :
+ ConsumerGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH;
commitRequestManager.updateMemberInformation(this.memberId, this.memberEpoch);
- currentAssignment = new HashSet<>();
+ currentAssignment = new HashMap<>();
transitionTo(MemberState.LEAVING);
}
@@ -638,7 +683,10 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
SortedSet<TopicPartition> assignedTopicPartitions = toTopicPartitionSet(assignedTopicIdPartitions);
// Check same assignment. Based on topic names for now, until topic IDs are properly
- // supported in the centralized subscription state object.
+ // supported in the centralized subscription state object. Note that this check is
+ // required to make sure that reconciliation is not triggered if the assignment ready to
+ // be reconciled is the same as the current one (even though the member may remain
+ // in RECONCILING state if it has some unresolved assignments).
boolean sameAssignmentReceived = assignedTopicPartitions.equals(ownedPartitions);
if (sameAssignmentReceived) {
@@ -681,22 +729,13 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
}
// Future that will complete when the full reconciliation process completes (revocation
- // and assignment, executed sequentially)
+ // and assignment, executed sequentially).
CompletableFuture<Void> reconciliationResult =
revocationResult.thenCompose(__ -> {
boolean memberHasRejoined = memberEpochOnReconciliationStart != memberEpoch;
if (state == MemberState.RECONCILING && !memberHasRejoined) {
// Apply assignment
- CompletableFuture<Void> assignResult = assignPartitions(assignedTopicPartitions,
- addedPartitions);
-
- // Clear topic names cache only for topics that are not in the subscription anymore
- for (TopicPartition tp : revokedPartitions) {
- if (!subscriptions.subscription().contains(tp.topic())) {
- assignedTopicNamesCache.values().remove(tp.topic());
- }
- }
- return assignResult;
+ return assignPartitions(assignedTopicIdPartitions, addedPartitions);
} else {
log.debug("Revocation callback completed but the member already " +
"transitioned out of the reconciling state for epoch {} into " +
@@ -724,12 +763,9 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
// Make assignment effective on the broker by transitioning to send acknowledge.
transitionTo(MemberState.ACKNOWLEDGING);
- // Make assignment effective on the member group manager
- currentAssignment = assignedTopicIdPartitions;
-
// Indicate that we completed reconciling a subset of the assignment ready to
// reconcile (new assignments might have been received or discovered in
- // metadata)
+ // metadata).
assignmentReadyToReconcile.removeAll(assignedTopicIdPartitions);
} else {
String reason = interruptedReconciliationErrorMessage();
@@ -742,6 +778,15 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
return true;
}
+ // Visible for testing.
+ void updateCurrentAssignment(Set<TopicIdPartition> assignedTopicIdPartitions) {
+ currentAssignment.clear();
+ assignedTopicIdPartitions.forEach(topicIdPartition -> {
+ Uuid topicId = topicIdPartition.topicId();
+ currentAssignment.computeIfAbsent(topicId, k -> new TreeSet<>()).add(topicIdPartition.partition());
+ });
+ }
+
/**
* Build set of {@link TopicPartition} from the given set of {@link TopicIdPartition}.
*/
@@ -800,18 +845,16 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
assignmentReadyToReconcile.clear();
// Try to resolve topic names from metadata cache or subscription cache, and move
// assignments from the "unresolved" collection, to the "readyToReconcile" one.
- Iterator<Map.Entry<Uuid, List<Integer>>> it = assignmentUnresolved.entrySet().iterator();
+ Iterator<Map.Entry<Uuid, SortedSet<Integer>>> it = assignmentUnresolved.entrySet().iterator();
while (it.hasNext()) {
- Map.Entry<Uuid, List<Integer>> e = it.next();
+ Map.Entry<Uuid, SortedSet<Integer>> e = it.next();
Uuid topicId = e.getKey();
- List<Integer> topicPartitions = e.getValue();
+ SortedSet<Integer> topicPartitions = e.getValue();
Optional<String> nameFromMetadata = findTopicNameInGlobalOrLocalCache(topicId);
nameFromMetadata.ifPresent(resolvedTopicName -> {
// Name resolved, so assignment is ready for reconciliation.
- SortedSet<TopicIdPartition> topicIdPartitions =
- buildAssignedPartitionsWithTopicName(topicId, resolvedTopicName, topicPartitions);
- assignmentReadyToReconcile.addAll(topicIdPartitions);
+ addToAssignmentReadyToReconcile(topicId, resolvedTopicName, topicPartitions);
it.remove();
});
}
@@ -847,22 +890,17 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
}
/**
- * Build set of TopicPartition for the partitions included in the heartbeat topicPartitions,
- * and using the given topic name.
+ * Build a TopicPartition for each of the partitions included in the heartbeat topicPartitions,
+ * and using the given topic name. Add the created TopicPartition to the
+ * {@link #assignmentReadyToReconcile}.
*/
- private SortedSet<TopicIdPartition> buildAssignedPartitionsWithTopicName(
- Uuid topicId,
- String topicName,
- List<Integer> topicPartitions) {
- SortedSet<TopicIdPartition> assignedPartitions =
- new TreeSet<>(TOPIC_ID_PARTITION_COMPARATOR);
+ private void addToAssignmentReadyToReconcile(Uuid topicId, String topicName, SortedSet<Integer> topicPartitions) {
topicPartitions.forEach(tp -> {
TopicIdPartition topicIdPartition = new TopicIdPartition(
topicId,
new TopicPartition(topicName, tp));
- assignedPartitions.add(topicIdPartition);
+ assignmentReadyToReconcile.add(topicIdPartition);
});
- return assignedPartitions;
}
/**
@@ -889,7 +927,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
markPendingRevocationToPauseFetching(revokedPartitions);
// Future that will complete when the revocation completes (including offset commit
- // request and user callback execution)
+ // request and user callback execution).
CompletableFuture<Void> revocationResult = new CompletableFuture<>();
// Commit offsets if auto-commit enabled.
@@ -909,6 +947,19 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
" proceed with the revocation anyway.", error);
}
+ // At this point we expect to be in a middle of a revocation triggered from RECONCILING
+ // or PREPARE_LEAVING, but it could be the case that the member received a fatal error
+ // while waiting for the commit to complete. Check if that's the case and abort the
+ // revocation.
+ if (state == MemberState.FATAL) {
+ String errorMsg = String.format("Member %s with epoch %s received a fatal error " +
+ "while waiting for a revocation commit to complete. Will abort revocation " +
+ "without triggering user callback.", memberId, memberEpoch);
+ log.debug(errorMsg);
+ revocationResult.completeExceptionally(new KafkaException(errorMsg));
+ return;
+ }
+
CompletableFuture<Void> userCallbackResult = invokeOnPartitionsRevokedCallback(revokedPartitions);
userCallbackResult.whenComplete((callbackResult, callbackError) -> {
if (callbackError != null) {
@@ -927,7 +978,8 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
/**
* Make new assignment effective and trigger onPartitionsAssigned callback for the partitions
- * added.
+ * added. This will also update the local topic names cache, removing from it all topics that
+ * are not assigned to the member anymore.
*
* @param assignedPartitions New assignment that will be updated in the member subscription
* state.
@@ -937,14 +989,20 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
* @return Future that will complete when the callback execution completes.
*/
private CompletableFuture<Void> assignPartitions(
- SortedSet<TopicPartition> assignedPartitions,
+ SortedSet<TopicIdPartition> assignedPartitions,
SortedSet<TopicPartition> addedPartitions) {
// Make assignment effective on the client by updating the subscription state.
updateSubscription(assignedPartitions, false);
- // Invoke user call back
- return invokeOnPartitionsAssignedCallback(addedPartitions);
+ // Invoke user call back.
+ CompletableFuture<Void> result = invokeOnPartitionsAssignedCallback(addedPartitions);
+
+ // Clear topic names cache, removing topics that are not assigned to the member anymore.
+ Set<String> assignedTopics = assignedPartitions.stream().map(TopicIdPartition::topic).collect(Collectors.toSet());
+ assignedTopicNamesCache.values().retainAll(assignedTopics);
+
+ return result;
}
/**
@@ -1047,7 +1105,7 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
* {@inheritDoc}
*/
@Override
- public Set<TopicIdPartition> currentAssignment() {
+ public Map<Uuid, SortedSet<Integer>> currentAssignment() {
return this.currentAssignment;
}
@@ -1090,7 +1148,6 @@ public class MembershipManagerImpl implements MembershipManager, ClusterResource
public void onUpdate(ClusterResource clusterResource) {
resolveMetadataForUnresolvedAssignment();
if (!assignmentReadyToReconcile.isEmpty()) {
- transitionTo(MemberState.RECONCILING);
reconcile();
}
}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java
index d39b63e9790..6fd10500669 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumerTest.java
@@ -16,7 +16,6 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import java.util.Locale;
import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
@@ -63,6 +62,7 @@ import org.apache.kafka.common.utils.Timer;
import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.params.ParameterizedTest;
@@ -81,6 +81,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
@@ -951,6 +952,7 @@ public class AsyncKafkaConsumerTest {
}
}
+ @Disabled("Flaky test temporarily disabled - in review")
@Test
public void testGroupIdNotNullAndValid() {
final Properties props = requiredConsumerPropertiesAndGroupId("consumerGroupA");
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java
index 53ee074e370..0496e90b2b4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java
@@ -18,6 +18,7 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
+import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
import org.apache.kafka.clients.consumer.internals.events.GroupMetadataUpdateEvent;
import org.apache.kafka.common.KafkaException;
@@ -71,6 +72,7 @@ public class HeartbeatRequestManagerTest {
private Time time;
private CoordinatorRequestManager coordinatorRequestManager;
private SubscriptionState subscriptions;
+ private Metadata metadata;
private HeartbeatRequestManager heartbeatRequestManager;
private MembershipManager membershipManager;
private HeartbeatRequestManager.HeartbeatRequestState heartbeatRequestState;
@@ -96,6 +98,7 @@ public class HeartbeatRequestManagerTest {
backgroundEventQueue = testBuilder.backgroundEventQueue;
subscriptions = testBuilder.subscriptions;
membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new);
+ metadata = testBuilder.metadata;
when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999)));
}
@@ -490,7 +493,8 @@ public class HeartbeatRequestManagerTest {
// Mock the response from the group coordinator which returns an assignment
ConsumerGroupHeartbeatResponseData.TopicPartitions tpTopic1 =
new ConsumerGroupHeartbeatResponseData.TopicPartitions();
- tpTopic1.setTopicId(Uuid.randomUuid());
+ Uuid topicId = Uuid.randomUuid();
+ tpTopic1.setTopicId(topicId);
tpTopic1.setPartitions(Collections.singletonList(0));
ConsumerGroupHeartbeatResponseData.Assignment assignmentTopic1 =
new ConsumerGroupHeartbeatResponseData.Assignment();
@@ -500,8 +504,9 @@ public class HeartbeatRequestManagerTest {
.setMemberId(memberId)
.setMemberEpoch(1)
.setAssignment(assignmentTopic1));
+ when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, "topic1"));
membershipManager.onHeartbeatResponseReceived(rs1.data());
- assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
}
private void mockStableMember() {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java
index 15f15057aae..30b1c1aea26 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MembershipManagerImplTest.java
@@ -23,6 +23,7 @@ import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData;
import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest;
import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -36,6 +37,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@@ -44,6 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyCollection;
import static org.mockito.ArgumentMatchers.anySet;
@@ -91,6 +95,15 @@ public class MembershipManagerImplTest {
return manager;
}
+ private MembershipManagerImpl createMembershipManagerJoiningGroup(String groupInstanceId) {
+ MembershipManagerImpl manager = spy(new MembershipManagerImpl(
+ GROUP_ID, Optional.ofNullable(groupInstanceId), Optional.empty(),
+ subscriptionState, commitRequestManager, metadata, testBuilder.logContext,
+ Optional.empty()));
+ manager.transitionToJoining();
+ return manager;
+ }
+
private MembershipManagerImpl createMembershipManagerJoiningGroup(String groupInstanceId,
String serverAssignor) {
MembershipManagerImpl manager = new MembershipManagerImpl(
@@ -223,8 +236,78 @@ public class MembershipManagerImplTest {
verify(subscriptionState).assignFromSubscribed(Collections.emptySet());
}
+ @Test
+ public void testFencingWhenStateIsPrepareLeaving() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+
+ // Start leaving group, blocked waiting for commit of all consumed to complete.
+ CompletableFuture<Void> commitResult = mockPrepareLeavingStuckCommitting();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.PREPARE_LEAVING, membershipManager.state());
+
+ // Get fenced while preparing to leave the group. Member should not trigger any
+ // callback or try to rejoin, and should continue leaving the group as it was
+ // before getting fenced.
+ MockRebalanceListener rebalanceListener = new MockRebalanceListener();
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.of(rebalanceListener));
+ membershipManager.transitionToFenced();
+ assertEquals(MemberState.PREPARE_LEAVING, membershipManager.state());
+ testFenceIsNoOp(membershipManager, rebalanceListener);
+
+ // When commit completes member should transition to LEAVE.
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty());
+ commitResult.complete(null);
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+ }
+
+ @Test
+ public void testFencingWhenStateIsLeaving() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+
+ // Start leaving group.
+ mockLeaveGroup();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+
+ // Get fenced while leaving. Member should not trigger any callback or try to
+ // rejoin, and should continue leaving the group as it was before getting fenced.
+ MockRebalanceListener rebalanceListener = new MockRebalanceListener();
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.of(rebalanceListener));
+ membershipManager.transitionToFenced();
+ testFenceIsNoOp(membershipManager, rebalanceListener);
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+
+ membershipManager.onHeartbeatRequestSent();
+ assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state());
+
+ // Last heartbeat sent is expected to fail, leading to a call to transitionToFenced. That
+ // should be no-op because the member already left.
+ membershipManager.transitionToFenced();
+ testFenceIsNoOp(membershipManager, rebalanceListener);
+ assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state());
+ }
+
+ @Test
+ public void testLeaveGroupEpoch() {
+ // Static member should leave the group with epoch -2.
+ MembershipManagerImpl membershipManager = createMemberInStableState("instance1");
+ mockLeaveGroup();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+ assertEquals(ConsumerGroupHeartbeatRequest.LEAVE_GROUP_STATIC_MEMBER_EPOCH,
+ membershipManager.memberEpoch());
+
+ // Dynamic member should leave the group with epoch -1.
+ membershipManager = createMemberInStableState(null);
+ mockLeaveGroup();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+ assertEquals(ConsumerGroupHeartbeatRequest.LEAVE_GROUP_MEMBER_EPOCH,
+ membershipManager.memberEpoch());
+ }
+
/**
- * This is the case where a member is stuck reconciling and transition out of the RECONCILING
+ * This is the case where a member is stuck reconciling and transitions out of the RECONCILING
* state (due to failure). When the reconciliation completes it should not be applied because
* it is not relevant anymore (it should not update the assignment on the member or send ack).
*/
@@ -233,23 +316,24 @@ public class MembershipManagerImplTest {
MembershipManagerImpl membershipManager = createMemberInStableState();
Uuid topicId1 = Uuid.randomUuid();
String topic1 = "topic1";
- Set<TopicPartition> owned = Collections.singleton(new TopicPartition(topic1, 0));
- mockOwnedPartitionAndAssignmentReceived(topicId1, topic1, owned, true);
+ List<TopicIdPartition> owned = Collections.singletonList(
+ new TopicIdPartition(topicId1, new TopicPartition(topic1, 0)));
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId1, topic1, owned, true);
// Reconciliation that does not complete stuck on revocation commit.
CompletableFuture<Void> commitResult = mockEmptyAssignmentAndRevocationStuckOnCommit(membershipManager);
- // Member received fatal error while reconciling
+ // Member received fatal error while reconciling.
when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true);
membershipManager.transitionToFatal();
verify(subscriptionState).assignFromSubscribed(Collections.emptySet());
clearInvocations(subscriptionState);
- // Complete commit request
+ // Complete commit request.
commitResult.complete(null);
// Member should not update the subscription or send ack when the delayed reconciliation
- // completed.
+ // completes.
verify(subscriptionState, never()).assignFromSubscribed(anySet());
assertNotEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
}
@@ -265,8 +349,9 @@ public class MembershipManagerImplTest {
MembershipManagerImpl membershipManager = createMemberInStableState();
Uuid topicId1 = Uuid.randomUuid();
String topic1 = "topic1";
- Set<TopicPartition> owned = Collections.singleton(new TopicPartition(topic1, 0));
- mockOwnedPartitionAndAssignmentReceived(topicId1, topic1, owned, true);
+ List<TopicIdPartition> owned = Collections.singletonList(new TopicIdPartition(topicId1,
+ new TopicPartition(topic1, 0)));
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId1, topic1, owned, true);
// Reconciliation that does not complete stuck on revocation commit.
CompletableFuture<Void> commitResult =
@@ -286,7 +371,7 @@ public class MembershipManagerImplTest {
// yet because there is another on in progress, but should keep the new assignment ready
// to be reconciled next.
Uuid topicId3 = Uuid.randomUuid();
- mockOwnedPartitionAndAssignmentReceived(topicId3, "topic3", owned, true);
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId3, "topic3", owned, true);
receiveAssignmentAfterRejoin(topicId3, Collections.singletonList(5), membershipManager);
verifyReconciliationNotTriggered(membershipManager);
Set<TopicIdPartition> assignmentAfterRejoin = Collections.singleton(
@@ -301,7 +386,6 @@ public class MembershipManagerImplTest {
// Assignment received after rejoining should be ready to reconcile on the next
// reconciliation loop.
- assertEquals(MemberState.RECONCILING, membershipManager.state());
assertEquals(assignmentAfterRejoin, membershipManager.assignmentReadyToReconcile());
}
@@ -375,6 +459,26 @@ public class MembershipManagerImplTest {
testLeaveGroupReleasesAssignmentAndResetsEpochToSendLeaveGroup(membershipManager);
}
+ @Test
+ public void testLeaveGroupWhenMemberOwnsAssignment() {
+ Uuid topicId = Uuid.randomUuid();
+ String topicName = "topic1";
+ MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.emptyList(), true);
+
+ receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager);
+
+ List<TopicIdPartition> assignedPartitions = Arrays.asList(
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 0)),
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 1)));
+ verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions);
+
+ assertTrue(membershipManager.currentAssignment().size() == 1);
+
+ testLeaveGroupReleasesAssignmentAndResetsEpochToSendLeaveGroup(membershipManager);
+ }
+
@Test
public void testLeaveGroupWhenMemberAlreadyLeaving() {
MembershipManager membershipManager = createMemberInStableState();
@@ -451,6 +555,64 @@ public class MembershipManagerImplTest {
testStateUpdateOnFatalFailure(membershipManager);
}
+ @Test
+ public void testFatalFailureWhenStateIsPrepareLeaving() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+
+ // Start leaving group, blocked waiting for commit of all consumed to complete.
+ CompletableFuture<Void> commitResult = mockPrepareLeavingStuckCommitting();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.PREPARE_LEAVING, membershipManager.state());
+
+ testStateUpdateOnFatalFailure(membershipManager);
+
+ // When commit completes member should abort the leave operation and remain in FATAL state.
+ commitResult.complete(null);
+ assertEquals(MemberState.FATAL, membershipManager.state());
+ }
+
+ @Test
+ public void testFatalFailureWhenStateIsLeaving() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+
+ // Start leaving group.
+ mockLeaveGroup();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+
+ // Get fatal failure while waiting to send the heartbeat to leave. Member should
+ // transition to FATAL, so the last heartbeat for leaving won't be sent because the member
+ // already failed.
+ testStateUpdateOnFatalFailure(membershipManager);
+
+ assertEquals(MemberState.FATAL, membershipManager.state());
+ membershipManager.onHeartbeatRequestSent();
+ assertEquals(MemberState.FATAL, membershipManager.state());
+ }
+
+ @Test
+ public void testFatalFailureWhenMemberAlreadyLeft() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+
+ // Start leaving group.
+ mockLeaveGroup();
+ membershipManager.leaveGroup();
+ assertEquals(MemberState.LEAVING, membershipManager.state());
+
+ // Last heartbeat sent.
+ membershipManager.onHeartbeatRequestSent();
+ assertEquals(MemberState.UNSUBSCRIBED, membershipManager.state());
+
+ // Fatal error received in response for the last heartbeat. Member should remain in FATAL
+ // state but no callbacks should be triggered because the member already left the group.
+ MockRebalanceListener rebalanceListener = new MockRebalanceListener();
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.of(rebalanceListener));
+ membershipManager.transitionToFatal();
+ assertEquals(0, rebalanceListener.lostCount);
+
+ assertEquals(MemberState.FATAL, membershipManager.state());
+ }
+
@Test
public void testUpdateStateFailsOnResponsesWithErrors() {
MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
@@ -473,8 +635,10 @@ public class MembershipManagerImplTest {
public void testNewAssignmentReplacesPreviousOneWaitingOnMetadata() {
MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false);
assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
- // When the ack is sent the member should go back to RECONCILING
+ // When the ack is sent nothing should change. Member still has nothing to reconcile,
+ // only topics waiting for metadata.
membershipManager.onHeartbeatRequestSent();
assertEquals(MemberState.RECONCILING, membershipManager.state());
assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
@@ -498,6 +662,82 @@ public class MembershipManagerImplTest {
assertTrue(membershipManager.topicsWaitingForMetadata().isEmpty());
}
+ /**
+ * This test ensures that member goes back to STABLE when the broker sends assignment that
+ * removes the unresolved target the client has, without triggering a reconciliation. In this
+ * case the member should discard the assignment that was unresolved and go back to STABLE with
+ * nothing to reconcile.
+ */
+ @Test
+ public void testNewEmptyAssignmentReplacesPreviousOneWaitingOnMetadata() {
+ MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false);
+ assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
+
+ // When the ack is sent nothing should change. Member still has nothing to reconcile,
+ // only topics waiting for metadata.
+ membershipManager.onHeartbeatRequestSent();
+ assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
+
+ // New target assignment received while there is another one waiting to be resolved
+ // and reconciled. This assignment does not include the previous one that is waiting
+ // for metadata, so the member will discard the topics that were waiting for metadata, and
+ // reconcile the new assignment.
+ Uuid topicId = Uuid.randomUuid();
+ String topicName = "topic1";
+ when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName));
+ receiveEmptyAssignment(membershipManager);
+ assertEquals(MemberState.STABLE, membershipManager.state());
+ verify(subscriptionState, never()).assignFromSubscribed(any());
+ }
+
+ @Test
+ public void testNewAssignmentNotInMetadataReplacesPreviousOneWaitingOnMetadata() {
+ MembershipManagerImpl membershipManager = mockJoinAndReceiveAssignment(false);
+ assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
+
+ // New target assignment (not found in metadata) received while there is another one
+ // waiting to be resolved and reconciled. This assignment does not include the previous
+ // one that is waiting for metadata, so the member will discard the topics that were
+ // waiting for metadata, and just keep the new one as unresolved.
+ Uuid topicId = Uuid.randomUuid();
+ when(metadata.topicNames()).thenReturn(Collections.emptyMap());
+ receiveAssignment(topicId, Collections.singletonList(0), membershipManager);
+ assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
+ assertEquals(topicId, membershipManager.topicsWaitingForMetadata().iterator().next());
+ }
+
+ /**
+ * This ensures that the client reconciles target assignments as soon as they are discovered
+ * in metadata, without depending on the broker to re-send the assignment.
+ */
+ @Test
+ public void testUnresolvedTargetAssignmentIsReconciledWhenMetadataReceived() {
+ MembershipManagerImpl membershipManager = createMemberInStableState();
+ // Assignment not in metadata. Member cannot reconcile it yet, but keeps it to be
+ // reconciled when metadata is discovered.
+ Uuid topicId = Uuid.randomUuid();
+ receiveAssignment(topicId, Collections.singletonList(1), membershipManager);
+ assertEquals(MemberState.RECONCILING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().size() > 0);
+
+ // Metadata update received, including the missing topic name.
+ String topicName = "topic1";
+ when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName));
+ when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true);
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty());
+ membershipManager.onUpdate(null);
+
+ // Assignment should have been reconciled.
+ Set<TopicPartition> expectedAssignment = Collections.singleton(new TopicPartition(topicName, 1));
+ verify(subscriptionState).assignFromSubscribed(expectedAssignment);
+ assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
+ assertTrue(membershipManager.topicsWaitingForMetadata().isEmpty());
+ }
+
/**
* This test should be the case when an assignment is sent to the member, and it cannot find
* it in metadata (temporarily). If the broker continues to send the assignment to the
@@ -545,14 +785,16 @@ public class MembershipManagerImplTest {
public void testReconcileNewPartitionsAssignedWhenNoPartitionOwned() {
Uuid topicId = Uuid.randomUuid();
String topicName = "topic1";
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true);
-
MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.emptyList(), true);
+
+
receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager);
- Set<TopicIdPartition> assignedPartitions = new HashSet<>(Arrays.asList(
- new TopicIdPartition(topicId, new TopicPartition(topicName, 0)),
- new TopicIdPartition(topicId, new TopicPartition(topicName, 1))));
+ List<TopicIdPartition> assignedPartitions = Arrays.asList(
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 0)),
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 1)));
verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions);
}
@@ -560,18 +802,18 @@ public class MembershipManagerImplTest {
public void testReconcileNewPartitionsAssignedWhenOtherPartitionsOwned() {
Uuid topicId = Uuid.randomUuid();
String topicName = "topic1";
- TopicPartition ownedPartition = new TopicPartition(topicName, 0);
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName,
- Collections.singleton(ownedPartition), true);
-
+ TopicIdPartition ownedPartition = new TopicIdPartition(topicId, new TopicPartition(topicName, 0));
MembershipManagerImpl membershipManager = createMemberInStableState();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.singletonList(ownedPartition), true);
+
+
// New assignment received, adding partitions 1 and 2 to the previously owned partition 0.
receiveAssignment(topicId, Arrays.asList(0, 1, 2), membershipManager);
- Set<TopicIdPartition> assignedPartitions = new HashSet<>(Arrays.asList(
- new TopicIdPartition(topicId, ownedPartition),
- new TopicIdPartition(topicId, new TopicPartition("topic1", 1)),
- new TopicIdPartition(topicId, new TopicPartition("topic1", 2))));
+ List<TopicIdPartition> assignedPartitions = Arrays.asList(ownedPartition,
+ new TopicIdPartition(topicId, new TopicPartition("topic1", 1)),
+ new TopicIdPartition(topicId, new TopicPartition("topic1", 2)));
verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions);
}
@@ -583,32 +825,33 @@ public class MembershipManagerImplTest {
String topicName = "topic1";
// Receive assignment different from what the member owns - should reconcile
- Set<TopicPartition> owned = Collections.emptySet();
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, owned, true);
- Set<TopicIdPartition> expectedAssignmentReconciled = new HashSet<>();
- expectedAssignmentReconciled.add(new TopicIdPartition(topicId,
- new TopicPartition(topicName, 0)));
- expectedAssignmentReconciled.add(new TopicIdPartition(topicId,
- new TopicPartition(topicName, 1)));
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName, Collections.emptyList(), true);
+ List<TopicIdPartition> expectedAssignmentReconciled = Arrays.asList(
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 0)),
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 1)));
receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager);
verifyReconciliationTriggeredAndCompleted(membershipManager, expectedAssignmentReconciled);
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
clearInvocations(subscriptionState, membershipManager);
+ membershipManager.onHeartbeatRequestSent();
+ assertEquals(MemberState.STABLE, membershipManager.state());
+
// Receive same assignment again - should not trigger reconciliation
- Set<TopicPartition> expectedTopicPartitionAssignment = buildTopicPartitions(expectedAssignmentReconciled);
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, expectedTopicPartitionAssignment, true);
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName, expectedAssignmentReconciled, true);
receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager);
// Verify new reconciliation was not triggered
verify(membershipManager, never()).markReconciliationInProgress();
verify(membershipManager, never()).markReconciliationCompleted();
verify(subscriptionState, never()).assignFromSubscribed(anyCollection());
+
+ assertEquals(MemberState.STABLE, membershipManager.state());
}
@Test
public void testReconcilePartitionsRevokedNoAutoCommitNoCallbacks() {
MembershipManagerImpl membershipManager = createMemberInStableState();
- mockOwnedPartition("topic1", 0);
+ mockOwnedPartition(membershipManager, Uuid.randomUuid(), "topic1", 0);
mockRevocationNoCallbacks(false);
@@ -620,7 +863,7 @@ public class MembershipManagerImplTest {
@Test
public void testReconcilePartitionsRevokedWithSuccessfulAutoCommitNoCallbacks() {
MembershipManagerImpl membershipManager = createMemberInStableState();
- mockOwnedPartition("topic1", 0);
+ mockOwnedPartition(membershipManager, Uuid.randomUuid(), "topic1", 0);
CompletableFuture<Void> commitResult = mockRevocationNoCallbacks(true);
@@ -641,7 +884,7 @@ public class MembershipManagerImplTest {
@Test
public void testReconcilePartitionsRevokedWithFailedAutoCommitCompletesRevocationAnyway() {
MembershipManagerImpl membershipManager = createMemberInStableState();
- mockOwnedPartition("topic1", 0);
+ mockOwnedPartition(membershipManager, Uuid.randomUuid(), "topic1", 0);
CompletableFuture<Void> commitResult = mockRevocationNoCallbacks(true);
@@ -663,19 +906,21 @@ public class MembershipManagerImplTest {
public void testReconcileNewPartitionsAssignedAndRevoked() {
Uuid topicId = Uuid.randomUuid();
String topicName = "topic1";
- TopicPartition ownedPartition = new TopicPartition(topicName, 0);
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.singleton(ownedPartition), true);
+ TopicIdPartition ownedPartition = new TopicIdPartition(topicId,
+ new TopicPartition(topicName, 0));
+ MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.singletonList(ownedPartition), true);
mockRevocationNoCallbacks(false);
- MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+
// New assignment received, revoking partition 0, and assigning new partitions 1 and 2.
receiveAssignment(topicId, Arrays.asList(1, 2), membershipManager);
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
- Set<TopicIdPartition> assignedPartitions = new HashSet<>(Arrays.asList(
- new TopicIdPartition(topicId, new TopicPartition("topic1", 1)),
- new TopicIdPartition(topicId, new TopicPartition("topic1", 2))));
+ Map<Uuid, SortedSet<Integer>> assignedPartitions = Collections.singletonMap(topicId,
+ new TreeSet<>(Arrays.asList(1, 2)));
assertEquals(assignedPartitions, membershipManager.currentAssignment());
assertFalse(membershipManager.reconciliationInProgress());
@@ -705,9 +950,9 @@ public class MembershipManagerImplTest {
// When metadata is updated, the member should re-trigger reconciliation
membershipManager.onUpdate(null);
- Set<TopicIdPartition> expectedAssignmentReconciled = new HashSet<>(Arrays.asList(
+ List<TopicIdPartition> expectedAssignmentReconciled = Arrays.asList(
new TopicIdPartition(topicId, new TopicPartition(topicName, 0)),
- new TopicIdPartition(topicId, new TopicPartition(topicName, 1))));
+ new TopicIdPartition(topicId, new TopicPartition(topicName, 1)));
verifyReconciliationTriggeredAndCompleted(membershipManager, expectedAssignmentReconciled);
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
assertTrue(membershipManager.topicsWaitingForMetadata().isEmpty());
@@ -745,19 +990,21 @@ public class MembershipManagerImplTest {
Uuid topicId = Uuid.randomUuid();
String topicName = "topic1";
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true);
+ MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.emptyList(), true);
// Member received assignment to reconcile;
- MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+
receiveAssignment(topicId, Arrays.asList(0, 1), membershipManager);
// Member should complete reconciliation
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
- Set<TopicPartition> assignedPartitions = new HashSet<>(Arrays.asList(
- new TopicPartition(topicName, 0),
- new TopicPartition(topicName, 1)));
- Set<TopicIdPartition> assignedTopicIdPartitions =
- assignedPartitions.stream().map(tp -> new TopicIdPartition(topicId, tp)).collect(Collectors.toSet());
+ List<Integer> partitions = Arrays.asList(0, 1);
+ Set<TopicPartition> assignedPartitions =
+ partitions.stream().map(p -> new TopicPartition(topicName, p)).collect(Collectors.toSet());
+ Map<Uuid, SortedSet<Integer>> assignedTopicIdPartitions = Collections.singletonMap(topicId,
+ new TreeSet<>(partitions));
assertEquals(assignedTopicIdPartitions, membershipManager.currentAssignment());
assertFalse(membershipManager.reconciliationInProgress());
@@ -775,7 +1022,7 @@ public class MembershipManagerImplTest {
// Revocation should complete without requesting any metadata update given that the topic
// received in target assignment should exist in local topic name cache.
verify(metadata, never()).requestUpdate(anyBoolean());
- Set<TopicIdPartition> remainingAssignment = Collections.singleton(
+ List<TopicIdPartition> remainingAssignment = Collections.singletonList(
new TopicIdPartition(topicId, new TopicPartition(topicName, 1)));
testRevocationCompleted(membershipManager, remainingAssignment);
@@ -790,17 +1037,24 @@ public class MembershipManagerImplTest {
verify(membershipManager, never()).transitionToJoining();
}
+ private void testFenceIsNoOp(MembershipManagerImpl membershipManager,
+ MockRebalanceListener rebalanceListener) {
+ assertNotEquals(0, membershipManager.memberEpoch());
+ assertEquals(0, rebalanceListener.lostCount);
+ assertEquals(0, rebalanceListener.revokedCount);
+ }
+
private MembershipManagerImpl mockMemberSuccessfullyReceivesAndAcksAssignment(
Uuid topicId, String topicName, List<Integer> partitions) {
- mockOwnedPartitionAndAssignmentReceived(topicId, topicName, Collections.emptySet(), true);
-
MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ mockOwnedPartitionAndAssignmentReceived(membershipManager, topicId, topicName,
+ Collections.emptyList(), true);
+
receiveAssignment(topicId, partitions, membershipManager);
- Set<TopicIdPartition> assignedPartitions = new HashSet<>();
- partitions.forEach(tp -> assignedPartitions.add(new TopicIdPartition(
- topicId,
- new TopicPartition(topicName, tp))));
+ List<TopicIdPartition> assignedPartitions =
+ partitions.stream().map(tp -> new TopicIdPartition(topicId,
+ new TopicPartition(topicName, tp))).collect(Collectors.toList());
verifyReconciliationTriggeredAndCompleted(membershipManager, assignedPartitions);
return membershipManager;
}
@@ -842,23 +1096,23 @@ public class MembershipManagerImplTest {
}
private void verifyReconciliationTriggeredAndCompleted(MembershipManagerImpl membershipManager,
- Set<TopicIdPartition> expectedAssignment) {
+ List<TopicIdPartition> expectedAssignment) {
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
verify(membershipManager).markReconciliationInProgress();
verify(membershipManager).markReconciliationCompleted();
assertFalse(membershipManager.reconciliationInProgress());
// Assignment applied
- Set<TopicPartition> expectedTopicPartitions = buildTopicPartitions(expectedAssignment);
- verify(subscriptionState).assignFromSubscribed(expectedTopicPartitions);
- // Set<TopicIdPartition> topicIdPartitions = buildTopicIdPartitions(expectedAssignment);
- assertEquals(expectedAssignment, membershipManager.currentAssignment());
+ List<TopicPartition> expectedTopicPartitions = buildTopicPartitions(expectedAssignment);
+ verify(subscriptionState).assignFromSubscribed(new HashSet<>(expectedTopicPartitions));
+ Map<Uuid, SortedSet<Integer>> assignmentByTopicId = assignmentByTopicId(expectedAssignment);
+ assertEquals(assignmentByTopicId, membershipManager.currentAssignment());
verify(commitRequestManager).resetAutoCommitTimer();
}
- private Set<TopicPartition> buildTopicPartitions(Set<TopicIdPartition> topicIdPartitions) {
- return topicIdPartitions.stream().map(TopicIdPartition::topicPartition).collect(Collectors.toSet());
+ private List<TopicPartition> buildTopicPartitions(List<TopicIdPartition> topicIdPartitions) {
+ return topicIdPartitions.stream().map(topicIdPartition -> topicIdPartition.topicPartition()).collect(Collectors.toList());
}
private void mockAckSent(MembershipManagerImpl membershipManager) {
@@ -895,26 +1149,38 @@ public class MembershipManagerImplTest {
}
private void testRevocationOfAllPartitionsCompleted(MembershipManagerImpl membershipManager) {
- testRevocationCompleted(membershipManager, Collections.emptySet());
+ testRevocationCompleted(membershipManager, Collections.emptyList());
}
private void testRevocationCompleted(MembershipManagerImpl membershipManager,
- Set<TopicIdPartition> expectedCurrentAssignment) {
+ List<TopicIdPartition> expectedCurrentAssignment) {
assertEquals(MemberState.ACKNOWLEDGING, membershipManager.state());
- assertEquals(expectedCurrentAssignment, membershipManager.currentAssignment());
+ Map<Uuid, SortedSet<Integer>> assignmentByTopicId = assignmentByTopicId(expectedCurrentAssignment);
+ assertEquals(assignmentByTopicId, membershipManager.currentAssignment());
assertFalse(membershipManager.reconciliationInProgress());
verify(subscriptionState).markPendingRevocation(anySet());
- Set<TopicPartition> expectedTopicPartitionAssignment =
+ List<TopicPartition> expectedTopicPartitionAssignment =
buildTopicPartitions(expectedCurrentAssignment);
- verify(subscriptionState).assignFromSubscribed(expectedTopicPartitionAssignment);
+ verify(subscriptionState).assignFromSubscribed(new HashSet<>(expectedTopicPartitionAssignment));
+ }
+
+ private Map<Uuid, SortedSet<Integer>> assignmentByTopicId(List<TopicIdPartition> topicIdPartitions) {
+ Map<Uuid, SortedSet<Integer>> assignmentByTopicId = new HashMap<>();
+ topicIdPartitions.forEach(topicIdPartition -> {
+ Uuid topicId = topicIdPartition.topicId();
+ assignmentByTopicId.computeIfAbsent(topicId, k -> new TreeSet<>()).add(topicIdPartition.partition());
+ });
+ return assignmentByTopicId;
}
- private void mockOwnedPartitionAndAssignmentReceived(Uuid topicId,
+ private void mockOwnedPartitionAndAssignmentReceived(MembershipManagerImpl membershipManager,
+ Uuid topicId,
String topicName,
- Set<TopicPartition> previouslyOwned,
+ List<TopicIdPartition> previouslyOwned,
boolean mockMetadata) {
- when(subscriptionState.assignedPartitions()).thenReturn(previouslyOwned);
+ when(subscriptionState.assignedPartitions()).thenReturn(getTopicPartitions(previouslyOwned));
+ membershipManager.updateCurrentAssignment(new HashSet<>(previouslyOwned));
if (mockMetadata) {
when(metadata.topicNames()).thenReturn(Collections.singletonMap(topicId, topicName));
}
@@ -922,8 +1188,17 @@ public class MembershipManagerImplTest {
when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty()).thenReturn(Optional.empty());
}
- private void mockOwnedPartition(String topic, int partition) {
+ private Set<TopicPartition> getTopicPartitions(List<TopicIdPartition> topicIdPartitions) {
+ return topicIdPartitions.stream().map(topicIdPartition ->
+ new TopicPartition(topicIdPartition.topic(), topicIdPartition.partition()))
+ .collect(Collectors.toSet());
+ }
+
+ private void mockOwnedPartition(MembershipManagerImpl membershipManager,
+ Uuid topicId, String topic, int partition) {
TopicPartition previouslyOwned = new TopicPartition(topic, partition);
+ membershipManager.updateCurrentAssignment(
+ Collections.singleton(new TopicIdPartition(topicId, new TopicPartition(topic, partition))));
when(subscriptionState.assignedPartitions()).thenReturn(Collections.singleton(previouslyOwned));
when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true);
}
@@ -950,7 +1225,11 @@ public class MembershipManagerImplTest {
}
private MembershipManagerImpl createMemberInStableState() {
- MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup();
+ return createMemberInStableState(null);
+ }
+
+ private MembershipManagerImpl createMemberInStableState(String groupInstanceId) {
+ MembershipManagerImpl membershipManager = createMembershipManagerJoiningGroup(groupInstanceId);
ConsumerGroupHeartbeatResponse heartbeatResponse = createConsumerGroupHeartbeatResponse(null);
membershipManager.onHeartbeatResponseReceived(heartbeatResponse.data());
assertEquals(MemberState.STABLE, membershipManager.state());
@@ -1031,13 +1310,26 @@ public class MembershipManagerImplTest {
doNothing().when(subscriptionState).markPendingRevocation(anySet());
}
- private void testStateUpdateOnFatalFailure(MembershipManager membershipManager) {
+ private CompletableFuture<Void> mockPrepareLeavingStuckCommitting() {
+ String topicName = "topic1";
+ TopicPartition ownedPartition = new TopicPartition(topicName, 0);
+ when(subscriptionState.assignedPartitions()).thenReturn(Collections.singleton(ownedPartition));
+ when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true);
+ when(subscriptionState.rebalanceListener()).thenReturn(Optional.empty());
+ doNothing().when(subscriptionState).markPendingRevocation(anySet());
+ when(commitRequestManager.autoCommitEnabled()).thenReturn(true);
+ CompletableFuture<Void> commitResult = new CompletableFuture<>();
+ when(commitRequestManager.maybeAutoCommitAllConsumed()).thenReturn(commitResult);
+ return commitResult;
+ }
+
+ private void testStateUpdateOnFatalFailure(MembershipManagerImpl membershipManager) {
String memberId = membershipManager.memberId();
int lastEpoch = membershipManager.memberEpoch();
when(subscriptionState.hasAutoAssignedPartitions()).thenReturn(true);
membershipManager.transitionToFatal();
assertEquals(MemberState.FATAL, membershipManager.state());
- // Should keep its last member id and epoch
+ // Should keep its last member id and epoch.
assertEquals(memberId, membershipManager.memberId());
assertEquals(lastEpoch, membershipManager.memberEpoch());
}