You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "clolov (via GitHub)" <gi...@apache.org> on 2023/04/13 11:45:57 UTC
[GitHub] [kafka] clolov commented on a diff in pull request #13538: KAFKA-14462; [8/N] Add ConsumerGroupMember
clolov commented on code in PR #13538:
URL: https://github.com/apache/kafka/pull/13538#discussion_r1165400494
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupMember.java:
##########
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.consumer;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.coordinator.group.generated.ConsumerGroupCurrentMemberAssignmentValue;
+import org.apache.kafka.coordinator.group.generated.ConsumerGroupMemberMetadataValue;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * ConsumerGroupMember contains all the information related to a member
+ * within a consumer group. This class is immutable and is fully backed
+ * by records stored in the __consumer_offsets topic.
+ */
+public class ConsumerGroupMember {
+ /**
+ * A builder allowing to create a new member or update an
+ * existing one.
+ *
+ * Please refer to the javadoc of {{@link ConsumerGroupMember}} for the
+ * definition of the fields.
+ */
+ public static class Builder {
+ private final String memberId;
+ private int memberEpoch = 0;
+ private int previousMemberEpoch = -1;
+ private int nextMemberEpoch = 0;
+ private String instanceId = null;
+ private String rackId = null;
+ private int rebalanceTimeoutMs = -1;
+ private String clientId = "";
+ private String clientHost = "";
+ private List<String> subscribedTopicNames = Collections.emptyList();
+ private String subscribedTopicRegex = "";
+ private String serverAssignorName = null;
+ private List<ClientAssignor> clientAssignors = Collections.emptyList();
+ private Map<Uuid, Set<Integer>> assignedPartitions = Collections.emptyMap();
+ private Map<Uuid, Set<Integer>> partitionsPendingRevocation = Collections.emptyMap();
+ private Map<Uuid, Set<Integer>> partitionsPendingAssignment = Collections.emptyMap();
+
+ public Builder(String memberId) {
+ this.memberId = Objects.requireNonNull(memberId);
+ }
+
+ public Builder(ConsumerGroupMember member) {
+ Objects.requireNonNull(member);
+
+ this.memberId = member.memberId;
+ this.memberEpoch = member.memberEpoch;
+ this.previousMemberEpoch = member.previousMemberEpoch;
+ this.nextMemberEpoch = member.nextMemberEpoch;
+ this.instanceId = member.instanceId;
+ this.rackId = member.rackId;
+ this.rebalanceTimeoutMs = member.rebalanceTimeoutMs;
+ this.clientId = member.clientId;
+ this.clientHost = member.clientHost;
+ this.subscribedTopicNames = member.subscribedTopicNames;
+ this.subscribedTopicRegex = member.subscribedTopicRegex;
+ this.serverAssignorName = member.serverAssignorName;
+ this.clientAssignors = member.clientAssignors;
+ this.assignedPartitions = member.assignedPartitions;
+ this.partitionsPendingRevocation = member.partitionsPendingRevocation;
+ this.partitionsPendingAssignment = member.partitionsPendingAssignment;
+ }
+
+ public Builder setMemberEpoch(int memberEpoch) {
+ this.memberEpoch = memberEpoch;
+ return this;
+ }
+
+ public Builder setPreviousMemberEpoch(int previousMemberEpoch) {
+ this.previousMemberEpoch = previousMemberEpoch;
+ return this;
+ }
+
+ public Builder setNextMemberEpoch(int nextMemberEpoch) {
+ this.nextMemberEpoch = nextMemberEpoch;
+ return this;
+ }
+
+ public Builder setInstanceId(String instanceId) {
+ this.instanceId = instanceId;
+ return this;
+ }
+
+ public Builder maybeUpdateInstanceId(Optional<String> instanceId) {
+ this.instanceId = instanceId.orElse(this.instanceId);
+ return this;
+ }
+
+ public Builder setRackId(String rackId) {
+ this.rackId = rackId;
+ return this;
+ }
+
+ public Builder maybeUpdateRackId(Optional<String> rackId) {
+ this.rackId = rackId.orElse(this.rackId);
+ return this;
+ }
+
+ public Builder setRebalanceTimeoutMs(int rebalanceTimeoutMs) {
+ this.rebalanceTimeoutMs = rebalanceTimeoutMs;
+ return this;
+ }
+
+ public Builder maybeUpdateRebalanceTimeoutMs(OptionalInt rebalanceTimeoutMs) {
+ this.rebalanceTimeoutMs = rebalanceTimeoutMs.orElse(this.rebalanceTimeoutMs);
+ return this;
+ }
+
+ public Builder setClientId(String clientId) {
+ this.clientId = clientId;
+ return this;
+ }
+
+ public Builder setClientHost(String clientHost) {
+ this.clientHost = clientHost;
+ return this;
+ }
+
+ public Builder setSubscribedTopicNames(List<String> subscribedTopicNames) {
+ this.subscribedTopicNames = subscribedTopicNames;
+ this.subscribedTopicNames.sort(Comparator.naturalOrder());
+ return this;
+ }
+
+ public Builder maybeUpdateSubscribedTopicNames(Optional<List<String>> subscribedTopicNames) {
+ this.subscribedTopicNames = subscribedTopicNames.orElse(this.subscribedTopicNames);
+ this.subscribedTopicNames.sort(Comparator.naturalOrder());
+ return this;
+ }
+
+ public Builder setSubscribedTopicRegex(String subscribedTopicRegex) {
+ this.subscribedTopicRegex = subscribedTopicRegex;
+ return this;
+ }
+
+ public Builder maybeUpdateSubscribedTopicRegex(Optional<String> subscribedTopicRegex) {
+ this.subscribedTopicRegex = subscribedTopicRegex.orElse(this.subscribedTopicRegex);
+ return this;
+ }
+
+ public Builder setServerAssignorName(String serverAssignorName) {
+ this.serverAssignorName = serverAssignorName;
+ return this;
+ }
+
+ public Builder maybeUpdateServerAssignorName(Optional<String> serverAssignorName) {
+ this.serverAssignorName = serverAssignorName.orElse(this.serverAssignorName);
+ return this;
+ }
+
+ public Builder setClientAssignors(List<ClientAssignor> clientAssignors) {
+ this.clientAssignors = clientAssignors;
+ return this;
+ }
+
+ public Builder maybeUpdateClientAssignors(Optional<List<ClientAssignor>> clientAssignors) {
+ this.clientAssignors = clientAssignors.orElse(this.clientAssignors);
+ return this;
+ }
+
+ public Builder setAssignedPartitions(Map<Uuid, Set<Integer>> assignedPartitions) {
+ this.assignedPartitions = assignedPartitions;
+ return this;
+ }
+
+ public Builder setPartitionsPendingRevocation(Map<Uuid, Set<Integer>> partitionsPendingRevocation) {
+ this.partitionsPendingRevocation = partitionsPendingRevocation;
+ return this;
+ }
+
+ public Builder setPartitionsPendingAssignment(Map<Uuid, Set<Integer>> partitionsPendingAssignment) {
+ this.partitionsPendingAssignment = partitionsPendingAssignment;
+ return this;
+ }
+
+ public Builder mergeWith(ConsumerGroupMemberMetadataValue record) {
Review Comment:
Isn't this more of a "replacing" operation than "merging"? If we were merging I would expect that for example the subscribedTopicNames would be appended rather than replaced.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/consumer/ConsumerGroupMember.java:
##########
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.consumer;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.coordinator.group.generated.ConsumerGroupCurrentMemberAssignmentValue;
+import org.apache.kafka.coordinator.group.generated.ConsumerGroupMemberMetadataValue;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * ConsumerGroupMember contains all the information related to a member
+ * within a consumer group. This class is immutable and is fully backed
+ * by records stored in the __consumer_offsets topic.
+ */
+public class ConsumerGroupMember {
+ /**
+ * A builder allowing to create a new member or update an
+ * existing one.
+ *
+ * Please refer to the javadoc of {{@link ConsumerGroupMember}} for the
+ * definition of the fields.
+ */
+ public static class Builder {
+ private final String memberId;
+ private int memberEpoch = 0;
+ private int previousMemberEpoch = -1;
+ private int nextMemberEpoch = 0;
+ private String instanceId = null;
+ private String rackId = null;
+ private int rebalanceTimeoutMs = -1;
+ private String clientId = "";
+ private String clientHost = "";
+ private List<String> subscribedTopicNames = Collections.emptyList();
+ private String subscribedTopicRegex = "";
+ private String serverAssignorName = null;
+ private List<ClientAssignor> clientAssignors = Collections.emptyList();
+ private Map<Uuid, Set<Integer>> assignedPartitions = Collections.emptyMap();
+ private Map<Uuid, Set<Integer>> partitionsPendingRevocation = Collections.emptyMap();
+ private Map<Uuid, Set<Integer>> partitionsPendingAssignment = Collections.emptyMap();
+
+ public Builder(String memberId) {
+ this.memberId = Objects.requireNonNull(memberId);
+ }
+
+ public Builder(ConsumerGroupMember member) {
+ Objects.requireNonNull(member);
+
+ this.memberId = member.memberId;
+ this.memberEpoch = member.memberEpoch;
+ this.previousMemberEpoch = member.previousMemberEpoch;
+ this.nextMemberEpoch = member.nextMemberEpoch;
+ this.instanceId = member.instanceId;
+ this.rackId = member.rackId;
+ this.rebalanceTimeoutMs = member.rebalanceTimeoutMs;
+ this.clientId = member.clientId;
+ this.clientHost = member.clientHost;
+ this.subscribedTopicNames = member.subscribedTopicNames;
+ this.subscribedTopicRegex = member.subscribedTopicRegex;
+ this.serverAssignorName = member.serverAssignorName;
+ this.clientAssignors = member.clientAssignors;
+ this.assignedPartitions = member.assignedPartitions;
+ this.partitionsPendingRevocation = member.partitionsPendingRevocation;
+ this.partitionsPendingAssignment = member.partitionsPendingAssignment;
+ }
+
+ public Builder setMemberEpoch(int memberEpoch) {
+ this.memberEpoch = memberEpoch;
+ return this;
+ }
+
+ public Builder setPreviousMemberEpoch(int previousMemberEpoch) {
+ this.previousMemberEpoch = previousMemberEpoch;
+ return this;
+ }
+
+ public Builder setNextMemberEpoch(int nextMemberEpoch) {
+ this.nextMemberEpoch = nextMemberEpoch;
+ return this;
+ }
+
+ public Builder setInstanceId(String instanceId) {
+ this.instanceId = instanceId;
+ return this;
+ }
+
+ public Builder maybeUpdateInstanceId(Optional<String> instanceId) {
+ this.instanceId = instanceId.orElse(this.instanceId);
+ return this;
+ }
+
+ public Builder setRackId(String rackId) {
+ this.rackId = rackId;
+ return this;
+ }
+
+ public Builder maybeUpdateRackId(Optional<String> rackId) {
+ this.rackId = rackId.orElse(this.rackId);
+ return this;
+ }
+
+ public Builder setRebalanceTimeoutMs(int rebalanceTimeoutMs) {
+ this.rebalanceTimeoutMs = rebalanceTimeoutMs;
+ return this;
+ }
+
+ public Builder maybeUpdateRebalanceTimeoutMs(OptionalInt rebalanceTimeoutMs) {
+ this.rebalanceTimeoutMs = rebalanceTimeoutMs.orElse(this.rebalanceTimeoutMs);
+ return this;
+ }
+
+ public Builder setClientId(String clientId) {
+ this.clientId = clientId;
+ return this;
+ }
+
+ public Builder setClientHost(String clientHost) {
+ this.clientHost = clientHost;
+ return this;
+ }
+
+ public Builder setSubscribedTopicNames(List<String> subscribedTopicNames) {
+ this.subscribedTopicNames = subscribedTopicNames;
+ this.subscribedTopicNames.sort(Comparator.naturalOrder());
+ return this;
+ }
+
+ public Builder maybeUpdateSubscribedTopicNames(Optional<List<String>> subscribedTopicNames) {
+ this.subscribedTopicNames = subscribedTopicNames.orElse(this.subscribedTopicNames);
+ this.subscribedTopicNames.sort(Comparator.naturalOrder());
+ return this;
+ }
+
+ public Builder setSubscribedTopicRegex(String subscribedTopicRegex) {
+ this.subscribedTopicRegex = subscribedTopicRegex;
+ return this;
+ }
+
+ public Builder maybeUpdateSubscribedTopicRegex(Optional<String> subscribedTopicRegex) {
+ this.subscribedTopicRegex = subscribedTopicRegex.orElse(this.subscribedTopicRegex);
+ return this;
+ }
+
+ public Builder setServerAssignorName(String serverAssignorName) {
+ this.serverAssignorName = serverAssignorName;
+ return this;
+ }
+
+ public Builder maybeUpdateServerAssignorName(Optional<String> serverAssignorName) {
+ this.serverAssignorName = serverAssignorName.orElse(this.serverAssignorName);
+ return this;
+ }
+
+ public Builder setClientAssignors(List<ClientAssignor> clientAssignors) {
+ this.clientAssignors = clientAssignors;
+ return this;
+ }
+
+ public Builder maybeUpdateClientAssignors(Optional<List<ClientAssignor>> clientAssignors) {
+ this.clientAssignors = clientAssignors.orElse(this.clientAssignors);
+ return this;
+ }
+
+ public Builder setAssignedPartitions(Map<Uuid, Set<Integer>> assignedPartitions) {
+ this.assignedPartitions = assignedPartitions;
+ return this;
+ }
+
+ public Builder setPartitionsPendingRevocation(Map<Uuid, Set<Integer>> partitionsPendingRevocation) {
+ this.partitionsPendingRevocation = partitionsPendingRevocation;
+ return this;
+ }
+
+ public Builder setPartitionsPendingAssignment(Map<Uuid, Set<Integer>> partitionsPendingAssignment) {
+ this.partitionsPendingAssignment = partitionsPendingAssignment;
+ return this;
+ }
+
+ public Builder mergeWith(ConsumerGroupMemberMetadataValue record) {
+ setInstanceId(record.instanceId());
+ setRackId(record.rackId());
+ setClientId(record.clientId());
+ setClientHost(record.clientHost());
+ setSubscribedTopicNames(record.subscribedTopicNames());
+ setSubscribedTopicRegex(record.subscribedTopicRegex());
+ setRebalanceTimeoutMs(record.rebalanceTimeoutMs());
+ setServerAssignorName(record.serverAssignor());
+ setClientAssignors(record.assignors().stream()
+ .map(ClientAssignor::fromRecord)
+ .collect(Collectors.toList()));
+ return this;
+ }
+
+ public Builder mergeWith(ConsumerGroupCurrentMemberAssignmentValue record) {
+ setMemberEpoch(record.memberEpoch());
+ setPreviousMemberEpoch(record.previousMemberEpoch());
+ setNextMemberEpoch(record.targetMemberEpoch());
+ setAssignedPartitions(assignmentFromTopicPartitions(record.assignedPartitions()));
+ setPartitionsPendingRevocation(assignmentFromTopicPartitions(record.partitionsPendingRevocation()));
+ setPartitionsPendingAssignment(assignmentFromTopicPartitions(record.partitionsPendingAssignment()));
+ return this;
+ }
+
+ private Map<Uuid, Set<Integer>> assignmentFromTopicPartitions(
+ List<ConsumerGroupCurrentMemberAssignmentValue.TopicPartitions> topicPartitionsList
+ ) {
+ return topicPartitionsList.stream().collect(Collectors.toMap(
+ ConsumerGroupCurrentMemberAssignmentValue.TopicPartitions::topicId,
+ topicPartitions -> Collections.unmodifiableSet(new HashSet<>(topicPartitions.partitions()))));
+ }
+
+ public ConsumerGroupMember build() {
+ MemberState state;
+ if (!partitionsPendingRevocation.isEmpty()) {
+ state = MemberState.REVOKING;
+ } else if (!partitionsPendingAssignment.isEmpty()) {
+ state = MemberState.ASSIGNING;
+ } else {
+ state = MemberState.STABLE;
+ }
+
+ return new ConsumerGroupMember(
+ memberId,
+ memberEpoch,
+ previousMemberEpoch,
+ nextMemberEpoch,
+ instanceId,
+ rackId,
+ rebalanceTimeoutMs,
+ clientId,
+ clientHost,
+ subscribedTopicNames,
+ subscribedTopicRegex,
+ serverAssignorName,
+ clientAssignors,
+ state,
+ assignedPartitions,
+ partitionsPendingRevocation,
+ partitionsPendingAssignment
+ );
+ }
+ }
+
+ public enum MemberState {
+ REVOKING("revoking"),
+ ASSIGNING("assigning"),
+ STABLE("stable");
+
+ private final String name;
+
+ MemberState(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ /**
+ * The member id.
+ */
+ private final String memberId;
+
+ /**
+ * The current member epoch.
+ */
+ private final int memberEpoch;
+
+ /**
+ * The previous member epoch.
+ */
+ private final int previousMemberEpoch;
+
+ /**
+ * The next member epoch. This corresponds to the target
+ * assignment epoch used to compute the current assigned,
+ * revoking and assigning partitions.
+ */
+ private final int nextMemberEpoch;
+
+ /**
+ * The instance id provided by the member.
+ */
+ private final String instanceId;
+
+ /**
+ * The rack id provided by the member.
+ */
+ private final String rackId;
+
+ /**
+ * The rebalance timeout provided by the member.
+ */
+ private final int rebalanceTimeoutMs;
+
+ /**
+ * The client id reported by the member.
+ */
+ private final String clientId;
+
+ /**
+ * The host reported by the member.
+ */
+ private final String clientHost;
+
+ /**
+ * The list of subscriptions (topic names) configured by the member.
+ */
+ private final List<String> subscribedTopicNames;
+
+ /**
+ * The subscription pattern configured by the member,
Review Comment:
```suggestion
* The subscription pattern configured by the member.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org