You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "lucasbru (via GitHub)" <gi...@apache.org> on 2025/01/28 16:18:15 UTC
[PR] KAFKA-18323: Add StreamsGroup class [kafka]
lucasbru opened a new pull request, #18729:
URL: https://github.com/apache/kafka/pull/18729
Implements a memory model for representing streams groups in the group coordinator, as well as group count and rebalance metrics.
This is a stacked PR. Only review the last commit.
### Committer Checklist (excluded from commit message)
- [ ] Verify design and implementation
- [ ] Verify test coverage and CI build status
- [ ] Verify documentation (including upgrade notes)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on PR #18729:
URL: https://github.com/apache/kafka/pull/18729#issuecomment-2633276629
Rebased on latest trunk
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1949796432
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
I am not sure if mocking the internal topic builder is such a good idea. The usage of the internal topic builder is an implementation detail that should not be exposed in a unit test, IMO. If we decide tomorrow to refactor how the configured topologies are computed we also need to change this unit test which actually is unnecessary since `StreamsGroup` does not have an explicit dependency (i.e. an injected dependency) to the internal topic builder.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "Copilot (via GitHub)" <gi...@apache.org>.
Copilot commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1939369884
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/topics/InternalTopicManager.java:
##########
@@ -74,12 +76,16 @@ public static ConfiguredTopology configureTopics(LogContext logContext,
Map<String, Integer> decidedPartitionCountsForInternalTopics =
decidePartitionCounts(logContext, topology, topicMetadata, copartitionGroupsBySubtopology, log);
- final Map<String, ConfiguredSubtopology> configuredSubtopologies =
+ final SortedMap<String, ConfiguredSubtopology> configuredSubtopologies =
subtopologies.stream()
.collect(Collectors.toMap(
StreamsGroupTopologyValue.Subtopology::subtopologyId,
- x -> fromPersistedSubtopology(x, decidedPartitionCountsForInternalTopics))
- );
+ x -> fromPersistedSubtopology(x, decidedPartitionCountsForInternalTopics),
+ (v1, v2) -> {
+ throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2));
Review Comment:
The error message for duplicate keys should include the key itself. Update the message to: throw new RuntimeException(String.format("Duplicate key %s for values %s and %s", key, v1, v2));
```suggestion
throw new RuntimeException(String.format("Duplicate key %s for values %s and %s", v1, v2));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru merged PR #18729:
URL: https://github.com/apache/kafka/pull/18729
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on PR #18729:
URL: https://github.com/apache/kafka/pull/18729#issuecomment-2630989500
Rebased on latest target assignment builder
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1951071719
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
OK, I see. I fine with the mock.
However, I still believe a test
> setTopology() with a previous call to setPartitionMetadata()
and without a subsequent call to `setPartitionMetadata()` would be helpful.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1949754882
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
Could you elaborate what did not make sense (for my own education)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1941239557
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
Review Comment:
The above comment also applies to this data structure.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
Review Comment:
Could you please add some more info to the javadoc? This data structure is a bit hard to grasp.
A visualization like the following would help:
```
subtopology -> partition -> memberId
```
I am wondering if using a class like `TaskId` for the key would make the code simpler.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
Review Comment:
What do you think of renaming this data structures to `currrentXTaskToProcessId`?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
Review Comment:
```suggestion
* Returns the current process ID of a task or null if the task does not have one.
```
Here and in other places.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
Review Comment:
What do you think of renaming this data structures to `currrentXTaskToMemberId`?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
Review Comment:
Does it make sense to change this to `Optional`?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
Review Comment:
```suggestion
* @param memberId The member ID.
```
Here and in other java docs.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
Review Comment:
```suggestion
* @return An immutable map containing all the target assignment keyed by member ID.
```
Here and elsewhere
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1950572107
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
> Could you elaborate what did not make sense (for my own education)?
You proposed to make these to cases a single test:
* setTopology() with a previous call to setPartitionMetadata()
* setPartitionMetadata() without a previous call to setTopology()
It seems the two cases will call different methods on the SUT and the expected result would be different (case 1 will initialize configured topology, case 2 will not initialized configured topology). Also, semantically, I just don't understand what makes those cases similar, so that they belong in the same test. Maybe you could explain (for my own education).
> I am not sure if mocking the internal topic builder is such a good idea. The usage of the internal topic builder is an implementation detail that should not be exposed in a unit test, IMO. If we decide tomorrow to refactor how the configured topologies are computed we also need to change this unit test which actually is unnecessary since StreamsGroup does not have an explicit dependency (i.e. an injected dependency) to the internal topic builder.
But the behavior of the internal topic builder is very complex. We do not want to test the entirety of its functionality in the streams group test. This stuff is being tested by the unit test of the internal topic builder - we want to exclude it from the unit that is being tested here. Mocking the internal topic builder seems ideal to me, since we can exclude it from the system under test without having to complicate the production code. That is, the unit test makes the dependency explicit, and this seems fine?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1952336102
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
Okay, I will address this in the next PR
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1939451460
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/topics/InternalTopicManager.java:
##########
@@ -74,12 +76,16 @@ public static ConfiguredTopology configureTopics(LogContext logContext,
Map<String, Integer> decidedPartitionCountsForInternalTopics =
decidePartitionCounts(logContext, topology, topicMetadata, copartitionGroupsBySubtopology, log);
- final Map<String, ConfiguredSubtopology> configuredSubtopologies =
+ final SortedMap<String, ConfiguredSubtopology> configuredSubtopologies =
subtopologies.stream()
.collect(Collectors.toMap(
StreamsGroupTopologyValue.Subtopology::subtopologyId,
- x -> fromPersistedSubtopology(x, decidedPartitionCountsForInternalTopics))
- );
+ x -> fromPersistedSubtopology(x, decidedPartitionCountsForInternalTopics),
+ (v1, v2) -> {
+ throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2));
Review Comment:
Suggestion is missing `key` :nerd_face:
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "bbejeck (via GitHub)" <gi...@apache.org>.
bbejeck commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1945188799
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
Review Comment:
When is this method called? How often is the offset updated? From the parameter name, I'm inferring that it occurs after every commit
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
Review Comment:
I agree with Bruno about a slight renaming for clarity.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1948733446
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1012 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group ID.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member ID.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner.
+ * The mapping is of the form <code>subtopology -> partition -> memberId</code>.
+ * When a member revokes a partition, it removes its process ID from this map.
+ * When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskToProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskToProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskToProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskToProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskToProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskToProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public Optional<ConfiguredTopology> configuredTopology() {
+ return configuredTopology.get();
+ }
+
+ public Optional<StreamsTopology> topology() {
+ return topology.get();
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group ID.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member ID of a static member that matches the given group instance ID.
+ *
+ * @param groupInstanceId The group instance ID.
+ * @return The member ID corresponding to the given instance ID or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member ID.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance ID.
+ * @return The member corresponding to the given instance ID or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member ID stored against the instance ID if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member ID to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member ID.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable map containing all the members keyed by their ID.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable map containing all the static members keyed by instance ID.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
Review Comment:
Done
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
I mocked the internal topic builder, to check if the partition metadata is actually passed correctly.
I tried your grouping - did not really make sense to me, but I attempted to extend the tests a bit.
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
Review Comment:
It's not independent. If I do not set the topology, I will transition to `NOT_READY`. If you do, you transition to `RECONCILING`. This is being tested here.
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
Review Comment:
It's not independent. If I do not set the topology & partiton metadata, I will transition to `NOT_READY`. If you do, you transition to `RECONCILING`. This is being tested here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1949781379
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
Review Comment:
Fair enough! I missed that connection.
However, would that verification not be more appropriate in `testUpdateMember`? My strategy would be to setup the state of the group, call the method under test, and then check how the state changed. In other words, I would only test the behavior of `setTopology()`. By calling `updateMember()` you also test the behavior of `updateMember()`, which IMO should be tested in a separate test, something like `testUpdateMemberWhenTopologyIsSet()` to better separate the concerns.
Said that, I am fine if you want to leave it like it is. I just wanted to explain my comment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1946916010
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
Review Comment:
Could you verify something more than just the existence of the configured topology? Like `configuredTopology.get().missingInternalTopics()`. Just to verify that the partition metadata was used to create the configured topology.
You could also consider merging this test with the previous test and verify the different orderings:
1. `setTopology()` without a previous call to `setPartitionMetadata()`
2. `setPartitionMetadata()` with a previous call to `setTopology()`
3. `setTopology()` with a previous call to `setPartitionMetadata()`
4. `setPartitionMetadata()` without a previous call to `setTopology()`
You could put cases 1 and 2 into one test and cases 3 and 4 into a different test.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1012 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group ID.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member ID.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner.
+ * The mapping is of the form <code>subtopology -> partition -> memberId</code>.
+ * When a member revokes a partition, it removes its process ID from this map.
+ * When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskToProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskToProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskToProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskToProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskToProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskToProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public Optional<ConfiguredTopology> configuredTopology() {
+ return configuredTopology.get();
+ }
+
+ public Optional<StreamsTopology> topology() {
+ return topology.get();
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group ID.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member ID of a static member that matches the given group instance ID.
+ *
+ * @param groupInstanceId The group instance ID.
+ * @return The member ID corresponding to the given instance ID or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member ID.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance ID.
+ * @return The member corresponding to the given instance ID or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member ID stored against the instance ID if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member ID to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member ID.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable map containing all the members keyed by their ID.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable map containing all the static members keyed by instance ID.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
Review Comment:
nit:
```suggestion
* @param memberId The member ID.
```
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
+ }
+
+ @Test
+ public void testSetPartitionMetadataUpdatesStateAndTopology() {
+ Uuid topicUuid = Uuid.randomUuid();
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+ streamsGroup.setTopology(topology);
+
+ Map<String, TopicMetadata> partitionMetadata = new HashMap<>();
+ partitionMetadata.put("topic1", new TopicMetadata(topicUuid, "topic1", 1));
+
+ streamsGroup.setPartitionMetadata(partitionMetadata);
+
+ assertEquals(partitionMetadata, streamsGroup.partitionMetadata());
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
Review Comment:
Also here, I do not understand this part.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
Review Comment:
The part of the `GroupMetadataManager` that calls this method is a bit weird. The group computes and returns the partition metadata so that the computed partition metadata can be passed back into the group. We should consider simplifying that part when we get to the `GroupMetadataManager`.
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,1073 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.ApiMessage;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupPartitionMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMemberKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTargetAssignmentMetadataKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyKey;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ ListGroupsResponseData.ListedGroup listedGroup = group.asListedGroup(1);
+
+ assertEquals("group-foo", listedGroup.groupId());
+ assertEquals("streams", listedGroup.protocolType());
+ assertEquals("Reconciling", listedGroup.groupState());
+ assertEquals("streams", listedGroup.groupType());
+ }
+
+ @Test
+ public void testValidateOffsetFetch() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ StreamsGroup group = new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ "group-foo",
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+
+ // Simulate a call from the admin client without member ID and member epoch.
+ group.validateOffsetFetch(null, -1, Long.MAX_VALUE);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE));
+
+ // Create a member.
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // The member does not exist at last committed offset 0.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetFetch("member-id", 0, 0));
+
+ // The member exists but the epoch is stale when the last committed offset is not considered.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetFetch("member-id", 10, Long.MAX_VALUE));
+
+ // This should succeed.
+ group.validateOffsetFetch("member-id", 0, Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testValidateDeleteGroup() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+ streamsGroup.updateMember(member1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ assertThrows(GroupNotEmptyException.class, streamsGroup::validateDeleteGroup);
+
+ streamsGroup.removeMember("member1");
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ assertDoesNotThrow(streamsGroup::validateDeleteGroup);
+ }
+
+ @Test
+ public void testOffsetExpirationCondition() {
+ long currentTimestamp = 30000L;
+ long commitTimestamp = 20000L;
+ long offsetsRetentionMs = 10000L;
+ OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(15000L, OptionalInt.empty(), "", commitTimestamp, OptionalLong.empty());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, new SnapshotRegistry(LOG_CONTEXT), "group-id", mock(GroupCoordinatorMetricsShard.class));
+
+ Optional<OffsetExpirationCondition> offsetExpirationCondition = group.offsetExpirationCondition();
+ assertTrue(offsetExpirationCondition.isPresent());
+
+ OffsetExpirationConditionImpl condition = (OffsetExpirationConditionImpl) offsetExpirationCondition.get();
+ assertEquals(commitTimestamp, condition.baseTimestamp().apply(offsetAndMetadata));
+ assertTrue(condition.isOffsetExpired(offsetAndMetadata, currentTimestamp, offsetsRetentionMs));
+ }
+
+ @Test
+ public void testAsDescribedGroup() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-id-1", mock(GroupCoordinatorMetricsShard.class));
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY.toString(), group.stateAsString(0));
+
+ group.setGroupEpoch(1);
+ group.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+ group.setTargetAssignmentEpoch(1);
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag1", "value1"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ group.updateMember(new StreamsGroupMember.Builder("member2")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setRebalanceTimeoutMs(1000)
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonMap("tag2", "value2"))
+ .setAssignedTasks(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .setTasksPendingRevocation(new TasksTuple(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()))
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+
+ StreamsGroupDescribeResponseData.DescribedGroup expected = new StreamsGroupDescribeResponseData.DescribedGroup()
+ .setGroupId("group-id-1")
+ .setGroupState(StreamsGroup.StreamsGroupState.STABLE.toString())
+ .setGroupEpoch(1)
+ .setTopology(new StreamsGroupDescribeResponseData.Topology().setEpoch(1).setSubtopologies(Collections.emptyList()))
+ .setAssignmentEpoch(1)
+ .setMembers(Arrays.asList(
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member1")
+ .setMemberEpoch(1)
+ .setInstanceId("instance1")
+ .setRackId("rack1")
+ .setClientId("client1")
+ .setClientHost("host1")
+ .setTopologyEpoch(1)
+ .setProcessId("process1")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host1").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag1").setValue("value1")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment()),
+ new StreamsGroupDescribeResponseData.Member()
+ .setMemberId("member2")
+ .setMemberEpoch(1)
+ .setInstanceId("instance2")
+ .setRackId("rack2")
+ .setClientId("client2")
+ .setClientHost("host2")
+ .setTopologyEpoch(1)
+ .setProcessId("process2")
+ .setUserEndpoint(new StreamsGroupDescribeResponseData.Endpoint().setHost("host2").setPort(9092))
+ .setClientTags(Collections.singletonList(new StreamsGroupDescribeResponseData.KeyValue().setKey("tag2").setValue("value2")))
+ .setAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ .setTargetAssignment(new StreamsGroupDescribeResponseData.Assignment())
+ ));
+ StreamsGroupDescribeResponseData.DescribedGroup actual = group.asDescribedGroup(1);
+
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testStateTransitionMetrics() {
+ // Confirm metrics is not updated when a new StreamsGroup is created but only when the group transitions
+ // its state.
+ GroupCoordinatorMetricsShard metrics = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(
+ LOG_CONTEXT,
+ new SnapshotRegistry(new LogContext()),
+ "group-id",
+ metrics
+ );
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(0)).onStreamsGroupStateTransition(null, StreamsGroup.StreamsGroupState.EMPTY);
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .setState(MemberState.STABLE)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.EMPTY, StreamsGroup.StreamsGroupState.NOT_READY);
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.NOT_READY, StreamsGroup.StreamsGroupState.RECONCILING);
+
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.RECONCILING, StreamsGroup.StreamsGroupState.ASSIGNING);
+
+ streamsGroup.setTargetAssignmentEpoch(1);
+
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.ASSIGNING, StreamsGroup.StreamsGroupState.STABLE);
+
+ streamsGroup.removeMember("member");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ verify(metrics, times(1)).onStreamsGroupStateTransition(StreamsGroup.StreamsGroupState.STABLE, StreamsGroup.StreamsGroupState.EMPTY);
+ }
+
+ @Test
+ public void testIsInStatesCaseInsensitiveAndUnderscored() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = new GroupCoordinatorMetricsShard(
+ snapshotRegistry,
+ emptyMap(),
+ new TopicPartition("__consumer_offsets", 0)
+ );
+ StreamsGroup group = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "group-foo", metricsShard);
+ snapshotRegistry.idempotentCreateSnapshot(0);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertFalse(group.isInStates(Collections.singleton("Empty"), 0));
+
+ group.updateMember(new StreamsGroupMember.Builder("member1")
+ .build());
+ snapshotRegistry.idempotentCreateSnapshot(1);
+ assertTrue(group.isInStates(Collections.singleton("empty"), 0));
+ assertTrue(group.isInStates(Collections.singleton("not_ready"), 1));
+ assertFalse(group.isInStates(Collections.singleton("empty"), 1));
+ }
+
+ @Test
+ public void testSetTopologyUpdatesConfiguredTopology() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+
+ StreamsTopology topology = new StreamsTopology(1, Collections.emptyMap());
+
+ streamsGroup.setTopology(topology);
+
+ Optional<ConfiguredTopology> configuredTopology = streamsGroup.configuredTopology();
+ assertTrue(configuredTopology.isPresent(), "Configured topology should be present");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ streamsGroup.updateMember(new StreamsGroupMember.Builder("member1")
+ .setMemberEpoch(1)
+ .build());
+
+ assertEquals(StreamsGroupState.RECONCILING, streamsGroup.state());
Review Comment:
What is the reason for adding a member and verifying the state of the group? They are independent of `setTopology()` that is tested here. I would end the test after line 935.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "lucasbru (via GitHub)" <gi...@apache.org>.
lucasbru commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1943676833
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
Review Comment:
Hmm, this is not used at all right now. In the consumer group, this is used to optimize the assignor, but we don't use it in our assignors. I think I will remove it for now, we can bring it back once we optimize the assignors.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
Review Comment:
Done
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
Review Comment:
Done
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
Review Comment:
Done
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,984 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testUpdateInvertedAssignment() {
Review Comment:
I removed the interverted assignment because we are not using it (yet)
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
+ TopicsImage topicsImage,
+ ClusterImage clusterImage,
+ StreamsTopology topology
+ ) {
+ Set<String> requiredTopicNames = topology.requiredTopics();
+
+ // Create the topic metadata for each subscribed topic.
+ Map<String, TopicMetadata> newSubscriptionMetadata = new HashMap<>(requiredTopicNames.size());
+
+ requiredTopicNames.forEach(topicName -> {
+ TopicImage topicImage = topicsImage.getTopic(topicName);
+ if (topicImage != null) {
+ Map<Integer, Set<String>> partitionRacks = new HashMap<>();
+ topicImage.partitions().forEach((partition, partitionRegistration) -> {
+ Set<String> racks = new HashSet<>();
+ for (int replica : partitionRegistration.replicas) {
+ Optional<String> rackOptional = clusterImage.broker(replica).rack();
+ // Only add the rack if it is available for the broker/replica.
+ rackOptional.ifPresent(racks::add);
+ }
+ // If rack information is unavailable for all replicas of this partition,
+ // no corresponding entry will be stored for it in the map.
+ if (!racks.isEmpty()) {
+ partitionRacks.put(partition, racks);
+ }
+ });
+
+ newSubscriptionMetadata.put(topicName, new TopicMetadata(
+ topicImage.id(),
+ topicImage.name(),
+ topicImage.partitions().size(),
+ partitionRacks)
+ );
+ }
+ });
+
+ return Collections.unmodifiableMap(newSubscriptionMetadata);
+ }
+
+ /**
+ * Updates the metadata refresh deadline.
+ *
+ * @param deadlineMs The deadline in milliseconds.
+ * @param groupEpoch The associated group epoch.
+ */
+ public void setMetadataRefreshDeadline(
+ long deadlineMs,
+ int groupEpoch
+ ) {
+ this.metadataRefreshDeadline = new DeadlineAndEpoch(deadlineMs, groupEpoch);
+ }
+
+ /**
+ * Requests a metadata refresh.
+ */
+ public void requestMetadataRefresh() {
+ this.metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+ }
+
+ /**
+ * Checks if a metadata refresh is required. A refresh is required in two cases: 1) The deadline is smaller or equal to the current
+ * time; 2) The group epoch associated with the deadline is larger than the current group epoch. This means that the operations which
+ * updated the deadline failed.
+ *
+ * @param currentTimeMs The current time in milliseconds.
+ * @return A boolean indicating whether a refresh is required or not.
+ */
+ public boolean hasMetadataExpired(long currentTimeMs) {
+ return currentTimeMs >= metadataRefreshDeadline.deadlineMs || groupEpoch() < metadataRefreshDeadline.epoch;
+ }
+
+ /**
+ * @return The metadata refresh deadline.
+ */
+ public DeadlineAndEpoch metadataRefreshDeadline() {
+ return metadataRefreshDeadline;
+ }
+
+ /**
+ * Validates the OffsetCommit request.
+ *
+ * @param memberId The member id.
+ * @param groupInstanceId The group instance id.
+ * @param memberEpoch The member epoch.
+ * @param isTransactional Whether the offset commit is transactional or not.
+ * @param apiVersion The api version.
+ * @throws UnknownMemberIdException If the member is not found.
+ * @throws StaleMemberEpochException If the provided member epoch doesn't match the actual member epoch.
+ */
+ @Override
+ public void validateOffsetCommit(
+ String memberId,
+ String groupInstanceId,
+ int memberEpoch,
+ boolean isTransactional,
+ short apiVersion
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member epoch is -1, the request comes from either the admin client
+ // or a consumer which does not use the group management facility. In this case,
+ // the request can commit offsets if the group is empty.
+ if (memberEpoch < 0 && members().isEmpty()) return;
+
+ // The TxnOffsetCommit API does not require the member id, the generation id and the group instance id fields.
+ // Hence, they are only validated if any of them is provided
+ if (isTransactional && memberEpoch == JoinGroupRequest.UNKNOWN_GENERATION_ID &&
+ memberId.equals(JoinGroupRequest.UNKNOWN_MEMBER_ID) && groupInstanceId == null)
+ return;
+
+ final StreamsGroupMember member = getOrMaybeCreateMember(memberId, false);
+
+ // If the commit is not transactional and the member uses the new streams protocol (KIP-1071),
+ // the member should be using the OffsetCommit API version >= 9.
+ if (!isTransactional && apiVersion < 9) {
+ throw new UnsupportedVersionException("OffsetCommit version 9 or above must be used " +
+ "by members using the streams group protocol");
+ }
+
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetFetch request.
+ *
+ * @param memberId The member id for streams groups.
+ * @param memberEpoch The member epoch for streams groups.
+ * @param lastCommittedOffset The last committed offsets in the timeline.
+ */
+ @Override
+ public void validateOffsetFetch(
+ String memberId,
+ int memberEpoch,
+ long lastCommittedOffset
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member id is null and the member epoch is -1, the request either comes
+ // from the admin client or from a client which does not provide them. In this case,
+ // the fetch request is accepted.
+ if (memberId == null && memberEpoch < 0) {
+ return;
+ }
+
+ final StreamsGroupMember member = members.get(memberId, lastCommittedOffset);
+ if (member == null) {
+ throw new UnknownMemberIdException(String.format("Member %s is not a member of group %s.",
+ memberId, groupId));
+ }
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetDelete request.
+ */
+ @Override
+ public void validateOffsetDelete() {
+ }
+
+ /**
+ * Validates the DeleteGroups request.
+ */
+ @Override
+ public void validateDeleteGroup() throws ApiException {
+ if (state() != StreamsGroupState.EMPTY) {
+ throw Errors.NON_EMPTY_GROUP.exception();
+ }
+ }
+
+ @Override
+ public boolean isSubscribedToTopic(String topic) {
+ return false;
Review Comment:
True. Fixed + a test
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
Review Comment:
I think this may have to checked in tight loops inside the assignor. I think it's a good place to avoid creating extra objects just for returning a value.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
Review Comment:
Let's try it
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
Review Comment:
Let's try it.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
Review Comment:
Done
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
Review Comment:
Done
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
Review Comment:
Done
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,984 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testUpdateInvertedAssignment() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+ String subtopologyId = "foo-sub";
+ String memberId1 = "member1";
+ String memberId2 = "member2";
+
+ // Initial assignment for member1
+ TasksTuple initialAssignment = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 0)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, initialAssignment);
+
+ // Verify that partition 0 is assigned to member1.
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(0, memberId1)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member1
+ TasksTuple newAssignment = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, newAssignment);
+
+ // Verify that partition 0 is no longer assigned and partition 1 is assigned to member1
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId1)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member2 to add partition 1
+ TasksTuple newAssignment2 = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId2, newAssignment2);
+
+ // Verify that partition 1 is assigned to member2
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId2)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member1 to revoke partition 1 and assign partition 0
+ TasksTuple newAssignment1 = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 0)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, newAssignment1);
+
+ // Verify that partition 1 is still assigned to member2 and partition 0 is assigned to member1
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(
+ mkEntry(0, memberId1),
+ mkEntry(1, memberId2)
+ ))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // Test remove target assignment for member1
+ streamsGroup.removeTargetAssignment(memberId1);
+
+ // Verify that partition 0 is no longer assigned and partition 1 is still assigned to member2
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId2)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member id and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member id and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
Review Comment:
Yes, you are right. This is what is being tested in the other groups as well, interestingly. I think what is being tested here is covered by other tests already. I replaced it by a test that makes sense to me.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
Review Comment:
Yes, let's postpone it.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
Review Comment:
Yes. I also realized that the rack information that we included here for future proofing (but not rack-aware assignment is currently done / planned) was removed from consumer groups due to the memory pressure it caused. I also updated this to not include the rack information. We can always bring it back in the future. This required some additional changes (but only removals).
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
Review Comment:
It's not really called. We always get the latest state, although we can get the state at every committed offset of the consumer offset topic. I removed it, we can always bring it back
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
+ TopicsImage topicsImage,
+ ClusterImage clusterImage,
+ StreamsTopology topology
+ ) {
+ Set<String> requiredTopicNames = topology.requiredTopics();
+
+ // Create the topic metadata for each subscribed topic.
+ Map<String, TopicMetadata> newSubscriptionMetadata = new HashMap<>(requiredTopicNames.size());
+
+ requiredTopicNames.forEach(topicName -> {
+ TopicImage topicImage = topicsImage.getTopic(topicName);
+ if (topicImage != null) {
+ Map<Integer, Set<String>> partitionRacks = new HashMap<>();
+ topicImage.partitions().forEach((partition, partitionRegistration) -> {
+ Set<String> racks = new HashSet<>();
+ for (int replica : partitionRegistration.replicas) {
+ Optional<String> rackOptional = clusterImage.broker(replica).rack();
+ // Only add the rack if it is available for the broker/replica.
+ rackOptional.ifPresent(racks::add);
+ }
+ // If rack information is unavailable for all replicas of this partition,
+ // no corresponding entry will be stored for it in the map.
+ if (!racks.isEmpty()) {
+ partitionRacks.put(partition, racks);
+ }
+ });
+
+ newSubscriptionMetadata.put(topicName, new TopicMetadata(
+ topicImage.id(),
+ topicImage.name(),
+ topicImage.partitions().size(),
+ partitionRacks)
+ );
+ }
+ });
+
+ return Collections.unmodifiableMap(newSubscriptionMetadata);
+ }
+
+ /**
+ * Updates the metadata refresh deadline.
+ *
+ * @param deadlineMs The deadline in milliseconds.
+ * @param groupEpoch The associated group epoch.
+ */
+ public void setMetadataRefreshDeadline(
+ long deadlineMs,
+ int groupEpoch
+ ) {
+ this.metadataRefreshDeadline = new DeadlineAndEpoch(deadlineMs, groupEpoch);
+ }
+
+ /**
+ * Requests a metadata refresh.
+ */
+ public void requestMetadataRefresh() {
+ this.metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+ }
+
+ /**
+ * Checks if a metadata refresh is required. A refresh is required in two cases: 1) The deadline is smaller or equal to the current
+ * time; 2) The group epoch associated with the deadline is larger than the current group epoch. This means that the operations which
+ * updated the deadline failed.
+ *
+ * @param currentTimeMs The current time in milliseconds.
+ * @return A boolean indicating whether a refresh is required or not.
+ */
+ public boolean hasMetadataExpired(long currentTimeMs) {
+ return currentTimeMs >= metadataRefreshDeadline.deadlineMs || groupEpoch() < metadataRefreshDeadline.epoch;
+ }
+
+ /**
+ * @return The metadata refresh deadline.
+ */
+ public DeadlineAndEpoch metadataRefreshDeadline() {
+ return metadataRefreshDeadline;
+ }
+
+ /**
+ * Validates the OffsetCommit request.
+ *
+ * @param memberId The member id.
+ * @param groupInstanceId The group instance id.
+ * @param memberEpoch The member epoch.
+ * @param isTransactional Whether the offset commit is transactional or not.
+ * @param apiVersion The api version.
+ * @throws UnknownMemberIdException If the member is not found.
+ * @throws StaleMemberEpochException If the provided member epoch doesn't match the actual member epoch.
+ */
+ @Override
+ public void validateOffsetCommit(
+ String memberId,
+ String groupInstanceId,
+ int memberEpoch,
+ boolean isTransactional,
+ short apiVersion
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member epoch is -1, the request comes from either the admin client
+ // or a consumer which does not use the group management facility. In this case,
+ // the request can commit offsets if the group is empty.
+ if (memberEpoch < 0 && members().isEmpty()) return;
+
+ // The TxnOffsetCommit API does not require the member id, the generation id and the group instance id fields.
+ // Hence, they are only validated if any of them is provided
+ if (isTransactional && memberEpoch == JoinGroupRequest.UNKNOWN_GENERATION_ID &&
+ memberId.equals(JoinGroupRequest.UNKNOWN_MEMBER_ID) && groupInstanceId == null)
+ return;
+
+ final StreamsGroupMember member = getOrMaybeCreateMember(memberId, false);
+
+ // If the commit is not transactional and the member uses the new streams protocol (KIP-1071),
+ // the member should be using the OffsetCommit API version >= 9.
+ if (!isTransactional && apiVersion < 9) {
+ throw new UnsupportedVersionException("OffsetCommit version 9 or above must be used " +
+ "by members using the streams group protocol");
+ }
+
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetFetch request.
+ *
+ * @param memberId The member id for streams groups.
+ * @param memberEpoch The member epoch for streams groups.
+ * @param lastCommittedOffset The last committed offsets in the timeline.
+ */
+ @Override
+ public void validateOffsetFetch(
+ String memberId,
+ int memberEpoch,
+ long lastCommittedOffset
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member id is null and the member epoch is -1, the request either comes
+ // from the admin client or from a client which does not provide them. In this case,
+ // the fetch request is accepted.
+ if (memberId == null && memberEpoch < 0) {
+ return;
+ }
+
+ final StreamsGroupMember member = members.get(memberId, lastCommittedOffset);
+ if (member == null) {
+ throw new UnknownMemberIdException(String.format("Member %s is not a member of group %s.",
+ memberId, groupId));
+ }
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetDelete request.
+ */
+ @Override
+ public void validateOffsetDelete() {
+ }
+
+ /**
+ * Validates the DeleteGroups request.
+ */
+ @Override
+ public void validateDeleteGroup() throws ApiException {
+ if (state() != StreamsGroupState.EMPTY) {
+ throw Errors.NON_EMPTY_GROUP.exception();
+ }
+ }
+
+ @Override
+ public boolean isSubscribedToTopic(String topic) {
+ return false;
+ }
+
+ /**
+ * Populates the list of records with tombstone(s) for deleting the group.
+ *
+ * @param records The list of records.
+ */
+ @Override
+ public void createGroupTombstoneRecords(List<CoordinatorRecord> records) {
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] KAFKA-18323: Add StreamsGroup class [kafka]
Posted by "cadonna (via GitHub)" <gi...@apache.org>.
cadonna commented on code in PR #18729:
URL: https://github.com/apache/kafka/pull/18729#discussion_r1942476981
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
Review Comment:
Could you please add a test that verifies that the configured topology is updated?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
Review Comment:
Would it make sense to return the `Optional` here?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
Review Comment:
Would it make sense to return the `Optional` here?
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,984 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testUpdateInvertedAssignment() {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ GroupCoordinatorMetricsShard metricsShard = mock(GroupCoordinatorMetricsShard.class);
+ StreamsGroup streamsGroup = new StreamsGroup(LOG_CONTEXT, snapshotRegistry, "test-group", metricsShard);
+ String subtopologyId = "foo-sub";
+ String memberId1 = "member1";
+ String memberId2 = "member2";
+
+ // Initial assignment for member1
+ TasksTuple initialAssignment = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 0)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, initialAssignment);
+
+ // Verify that partition 0 is assigned to member1.
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(0, memberId1)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member1
+ TasksTuple newAssignment = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, newAssignment);
+
+ // Verify that partition 0 is no longer assigned and partition 1 is assigned to member1
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId1)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member2 to add partition 1
+ TasksTuple newAssignment2 = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId2, newAssignment2);
+
+ // Verify that partition 1 is assigned to member2
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId2)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // New assignment for member1 to revoke partition 1 and assign partition 0
+ TasksTuple newAssignment1 = new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(subtopologyId, 0)),
+ emptyMap(),
+ emptyMap()
+ );
+ streamsGroup.updateTargetAssignment(memberId1, newAssignment1);
+
+ // Verify that partition 1 is still assigned to member2 and partition 0 is assigned to member1
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(
+ mkEntry(0, memberId1),
+ mkEntry(1, memberId2)
+ ))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+
+ // Test remove target assignment for member1
+ streamsGroup.removeTargetAssignment(memberId1);
+
+ // Verify that partition 0 is no longer assigned and partition 1 is still assigned to member2
+ assertEquals(
+ mkMap(
+ mkEntry(subtopologyId, mkMap(mkEntry(1, memberId2)))
+ ),
+ streamsGroup.invertedTargetActiveTasksAssignment()
+ );
+ }
+
+ @Test
+ public void testMetadataRefreshDeadline() {
+ MockTime time = new MockTime();
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Group epoch starts at 0.
+ assertEquals(0, group.groupEpoch());
+
+ // The refresh time deadline should be empty when the group is created or loaded.
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Advance past the deadline. The metadata should have expired.
+ time.sleep(1001L);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+
+ // Set the refresh time deadline with a higher group epoch. The metadata is considered
+ // as expired because the group epoch attached to the deadline is higher than the
+ // current group epoch.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch() + 1);
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch() + 1, group.metadataRefreshDeadline().epoch);
+
+ // Advance the group epoch.
+ group.setGroupEpoch(group.groupEpoch() + 1);
+
+ // Set the refresh deadline. The metadata remains valid because the deadline
+ // has not past and the group epoch is correct.
+ group.setMetadataRefreshDeadline(time.milliseconds() + 1000, group.groupEpoch());
+ assertFalse(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(time.milliseconds() + 1000, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(group.groupEpoch(), group.metadataRefreshDeadline().epoch);
+
+ // Request metadata refresh. The metadata expires immediately.
+ group.requestMetadataRefresh();
+ assertTrue(group.hasMetadataExpired(time.milliseconds()));
+ assertEquals(0L, group.metadataRefreshDeadline().deadlineMs);
+ assertEquals(0, group.metadataRefreshDeadline().epoch);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.TXN_OFFSET_COMMIT)
+ public void testValidateTransactionalOffsetCommit(short version) {
+ boolean isTransactional = true;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+
+ // Simulate a call from the admin client without member id and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create a member.
+ group.updateMember(new StreamsGroupMember.Builder("member-id").setMemberEpoch(0).build());
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+
+ // The member epoch is stale.
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("member-id", "", 10, isTransactional, version));
+
+ // This should succeed.
+ group.validateOffsetCommit("member-id", "", 0, isTransactional, version);
+
+ // This should succeed.
+ group.validateOffsetCommit("", null, -1, isTransactional, version);
+ }
+
+ @ParameterizedTest
+ @ApiKeyVersionsSource(apiKey = ApiKeys.OFFSET_COMMIT)
+ public void testValidateOffsetCommit(short version) {
+ boolean isTransactional = false;
+ StreamsGroup group = createStreamsGroup("group-foo");
+
+ // Simulate a call from the admin client without member id and member epoch.
+ // This should pass only if the group is empty.
+ group.validateOffsetCommit("", "", -1, isTransactional, version);
+
+ // The member does not exist.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("member-id", null, 0, isTransactional, version));
+
+ // Create members.
+ group.updateMember(
+ new StreamsGroupMember
+ .Builder("new-protocol-member-id").setMemberEpoch(0).build()
+ );
+
+ // A call from the admin client should fail as the group is not empty.
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", "", -1, isTransactional, version));
+ assertThrows(UnknownMemberIdException.class, () ->
+ group.validateOffsetCommit("", null, -1, isTransactional, version));
+
+ // The member epoch is stale.
+ if (version >= 9) {
+ assertThrows(StaleMemberEpochException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 10, isTransactional, version));
+ }
+
+ // This should succeed.
+ if (version >= 9) {
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version);
+ } else {
+ assertThrows(UnsupportedVersionException.class, () ->
+ group.validateOffsetCommit("new-protocol-member-id", "", 0, isTransactional, version));
+ }
+ }
+
+ @Test
+ public void testAsListedGroup() {
Review Comment:
I do not understand this test. The test name is `testAsListedGroup()` but the method `asListedGroup()` is never called.
##########
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java:
##########
@@ -0,0 +1,984 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.GroupNotEmptyException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource;
+import org.apache.kafka.coordinator.group.OffsetAndMetadata;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState;
+import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole;
+import org.apache.kafka.timeline.SnapshotRegistry;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.common.utils.Utils.mkEntry;
+import static org.apache.kafka.common.utils.Utils.mkMap;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksPerSubtopology;
+import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class StreamsGroupTest {
+
+ private static final LogContext LOG_CONTEXT = new LogContext();
+
+ private StreamsGroup createStreamsGroup(String groupId) {
+ SnapshotRegistry snapshotRegistry = new SnapshotRegistry(LOG_CONTEXT);
+ return new StreamsGroup(
+ LOG_CONTEXT,
+ snapshotRegistry,
+ groupId,
+ mock(GroupCoordinatorMetricsShard.class)
+ );
+ }
+
+ @Test
+ public void testGetOrCreateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ // Create a member.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", true);
+ assertEquals("member-id", member.memberId());
+
+ // Add member to the group.
+ streamsGroup.updateMember(member);
+
+ // Get that member back.
+ member = streamsGroup.getOrMaybeCreateMember("member-id", false);
+ assertEquals("member-id", member.memberId());
+
+ assertThrows(UnknownMemberIdException.class, () ->
+ streamsGroup.getOrMaybeCreateMember("does-not-exist", false));
+ }
+
+ @Test
+ public void testUpdateMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member).build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ }
+
+ @Test
+ public void testNoStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Create a new member which is not static
+ streamsGroup.getOrMaybeCreateMember("member", true);
+ assertNull(streamsGroup.staticMember("instance-id"));
+ }
+
+ @Test
+ public void testGetStaticMemberByInstanceId() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = streamsGroup.getOrMaybeCreateMember("member", true);
+
+ member = new StreamsGroupMember.Builder(member)
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals(member, streamsGroup.staticMember("instance"));
+ assertEquals(member, streamsGroup.getOrMaybeCreateMember("member", false));
+ assertEquals(member.memberId(), streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testRemoveMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = streamsGroup.getOrMaybeCreateMember("member", true);
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+
+ }
+
+ @Test
+ public void testRemoveStaticMember() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember member = new StreamsGroupMember.Builder("member")
+ .setInstanceId("instance")
+ .build();
+
+ streamsGroup.updateMember(member);
+ assertTrue(streamsGroup.hasMember("member"));
+
+ streamsGroup.removeMember("member");
+ assertFalse(streamsGroup.hasMember("member"));
+ assertNull(streamsGroup.staticMember("instance"));
+ assertNull(streamsGroup.staticMemberId("instance"));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process1"),
+ streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(),
+ streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsReassignedBeforeBeingRevoked() {
+ String fooSubtopologyId = "foo-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ emptyMap(),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+
+ member = new StreamsGroupMember.Builder(member)
+ .setProcessId("process1")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ )
+ )
+ .setTasksPendingRevocation(TasksTuple.EMPTY)
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process1", streamsGroup.currentActiveTaskProcessId(fooSubtopologyId, 1));
+ }
+
+ @Test
+ public void testUpdatingMemberUpdatesTaskProcessIdWhenPartitionIsNotReleased() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ StreamsGroupMember m2 = new StreamsGroupMember.Builder("m2")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ emptyMap(),
+ emptyMap()
+ )
+ )
+ .build();
+
+ // m2 should not be able to acquire foo-1 because the partition is
+ // still owned by another member.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.updateMember(m2));
+ }
+
+
+ @ParameterizedTest
+ @EnumSource(TaskRole.class)
+ public void testRemoveTaskProcessIds(TaskRole taskRole) {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ // Removing should fail because there is no epoch set.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process"
+ ));
+
+ StreamsGroupMember m1 = new StreamsGroupMember.Builder("m1")
+ .setProcessId("process")
+ .setAssignedTasks(mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)))
+ .build();
+
+ streamsGroup.updateMember(m1);
+
+ // Removing should fail because the expected epoch is incorrect.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.removeTaskProcessIds(
+ mkTasksTuple(taskRole, mkTasks(fooSubtopologyId, 1)),
+ "process1"
+ ));
+ }
+
+ @Test
+ public void testAddTaskProcessIds() {
+ String fooSubtopologyId = "foo-sub";
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+
+ streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ );
+
+ // Changing the epoch should fail because the owner of the partition
+ // should remove it first.
+ assertThrows(IllegalStateException.class, () -> streamsGroup.addTaskProcessId(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopologyId, 3))
+ ),
+ "process"
+ ));
+ }
+
+ @Test
+ public void testDeletingMemberRemovesProcessId() {
+ String fooSubtopology = "foo-sub";
+ String barSubtopology = "bar-sub";
+ String zarSubtopology = "zar-sub";
+
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ StreamsGroupMember member;
+
+ member = new StreamsGroupMember.Builder("member")
+ .setProcessId("process")
+ .setAssignedTasks(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 1)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 2)),
+ mkTasksPerSubtopology(mkTasks(fooSubtopology, 3))
+ )
+ )
+ .setTasksPendingRevocation(
+ new TasksTuple(
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 4)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 5)),
+ mkTasksPerSubtopology(mkTasks(barSubtopology, 6))
+ )
+ )
+ .build();
+
+ streamsGroup.updateMember(member);
+
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(fooSubtopology, 1));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(fooSubtopology, 2));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(fooSubtopology, 3));
+ assertEquals("process", streamsGroup.currentActiveTaskProcessId(barSubtopology, 4));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentStandbyTaskProcessIds(barSubtopology, 5));
+ assertEquals(Collections.singleton("process"), streamsGroup.currentWarmupTaskProcessIds(barSubtopology, 6));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+
+ streamsGroup.removeMember(member.memberId());
+
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 1));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 2));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 3));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 3));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 4));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 5));
+ assertNull(streamsGroup.currentActiveTaskProcessId(zarSubtopology, 7));
+ assertEquals(Collections.emptySet(), streamsGroup.currentStandbyTaskProcessIds(zarSubtopology, 8));
+ assertEquals(Collections.emptySet(), streamsGroup.currentWarmupTaskProcessIds(zarSubtopology, 9));
+ }
+
+ @Test
+ public void testGroupState() {
+ StreamsGroup streamsGroup = createStreamsGroup("foo");
+ assertEquals(StreamsGroupState.EMPTY, streamsGroup.state());
+
+ StreamsGroupMember member1 = new StreamsGroupMember.Builder("member1")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member1);
+ streamsGroup.setGroupEpoch(1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.NOT_READY, streamsGroup.state());
+
+ streamsGroup.setTopology(new StreamsTopology(1, Collections.emptyMap()));
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ StreamsGroupMember member2 = new StreamsGroupMember.Builder("member2")
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(1)
+ .setPreviousMemberEpoch(0)
+ .build();
+
+ streamsGroup.updateMember(member2);
+ streamsGroup.setGroupEpoch(2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.ASSIGNING, streamsGroup.state());
+
+ streamsGroup.setTargetAssignmentEpoch(2);
+
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member1 = new StreamsGroupMember.Builder(member1)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member1);
+
+ assertEquals(MemberState.STABLE, member1.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ // Member 2 is not stable so the group stays in reconciling state.
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.UNREVOKED_TASKS)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.UNREVOKED_TASKS, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.RECONCILING, streamsGroup.state());
+
+ member2 = new StreamsGroupMember.Builder(member2)
+ .setState(MemberState.STABLE)
+ .setMemberEpoch(2)
+ .setPreviousMemberEpoch(1)
+ .build();
+
+ streamsGroup.updateMember(member2);
+
+ assertEquals(MemberState.STABLE, member2.state());
+ assertEquals(StreamsGroup.StreamsGroupState.STABLE, streamsGroup.state());
+
+ streamsGroup.removeMember("member1");
+ streamsGroup.removeMember("member2");
+
+ assertEquals(StreamsGroup.StreamsGroupState.EMPTY, streamsGroup.state());
+ }
+
+ @Test
+ public void testUpdateInvertedAssignment() {
Review Comment:
Could you please also add a case where the subtopology ID is modified?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
+ TopicsImage topicsImage,
+ ClusterImage clusterImage,
+ StreamsTopology topology
+ ) {
+ Set<String> requiredTopicNames = topology.requiredTopics();
+
+ // Create the topic metadata for each subscribed topic.
+ Map<String, TopicMetadata> newSubscriptionMetadata = new HashMap<>(requiredTopicNames.size());
+
+ requiredTopicNames.forEach(topicName -> {
+ TopicImage topicImage = topicsImage.getTopic(topicName);
+ if (topicImage != null) {
+ Map<Integer, Set<String>> partitionRacks = new HashMap<>();
+ topicImage.partitions().forEach((partition, partitionRegistration) -> {
+ Set<String> racks = new HashSet<>();
+ for (int replica : partitionRegistration.replicas) {
+ Optional<String> rackOptional = clusterImage.broker(replica).rack();
+ // Only add the rack if it is available for the broker/replica.
+ rackOptional.ifPresent(racks::add);
+ }
+ // If rack information is unavailable for all replicas of this partition,
+ // no corresponding entry will be stored for it in the map.
+ if (!racks.isEmpty()) {
+ partitionRacks.put(partition, racks);
+ }
+ });
+
+ newSubscriptionMetadata.put(topicName, new TopicMetadata(
+ topicImage.id(),
+ topicImage.name(),
+ topicImage.partitions().size(),
+ partitionRacks)
+ );
+ }
+ });
+
+ return Collections.unmodifiableMap(newSubscriptionMetadata);
+ }
+
+ /**
+ * Updates the metadata refresh deadline.
+ *
+ * @param deadlineMs The deadline in milliseconds.
+ * @param groupEpoch The associated group epoch.
+ */
+ public void setMetadataRefreshDeadline(
+ long deadlineMs,
+ int groupEpoch
+ ) {
+ this.metadataRefreshDeadline = new DeadlineAndEpoch(deadlineMs, groupEpoch);
+ }
+
+ /**
+ * Requests a metadata refresh.
+ */
+ public void requestMetadataRefresh() {
+ this.metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+ }
+
+ /**
+ * Checks if a metadata refresh is required. A refresh is required in two cases: 1) The deadline is smaller or equal to the current
+ * time; 2) The group epoch associated with the deadline is larger than the current group epoch. This means that the operations which
+ * updated the deadline failed.
+ *
+ * @param currentTimeMs The current time in milliseconds.
+ * @return A boolean indicating whether a refresh is required or not.
+ */
+ public boolean hasMetadataExpired(long currentTimeMs) {
+ return currentTimeMs >= metadataRefreshDeadline.deadlineMs || groupEpoch() < metadataRefreshDeadline.epoch;
+ }
+
+ /**
+ * @return The metadata refresh deadline.
+ */
+ public DeadlineAndEpoch metadataRefreshDeadline() {
+ return metadataRefreshDeadline;
+ }
+
+ /**
+ * Validates the OffsetCommit request.
+ *
+ * @param memberId The member id.
+ * @param groupInstanceId The group instance id.
+ * @param memberEpoch The member epoch.
+ * @param isTransactional Whether the offset commit is transactional or not.
+ * @param apiVersion The api version.
+ * @throws UnknownMemberIdException If the member is not found.
+ * @throws StaleMemberEpochException If the provided member epoch doesn't match the actual member epoch.
+ */
+ @Override
+ public void validateOffsetCommit(
+ String memberId,
+ String groupInstanceId,
+ int memberEpoch,
+ boolean isTransactional,
+ short apiVersion
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member epoch is -1, the request comes from either the admin client
+ // or a consumer which does not use the group management facility. In this case,
+ // the request can commit offsets if the group is empty.
+ if (memberEpoch < 0 && members().isEmpty()) return;
+
+ // The TxnOffsetCommit API does not require the member id, the generation id and the group instance id fields.
+ // Hence, they are only validated if any of them is provided
+ if (isTransactional && memberEpoch == JoinGroupRequest.UNKNOWN_GENERATION_ID &&
+ memberId.equals(JoinGroupRequest.UNKNOWN_MEMBER_ID) && groupInstanceId == null)
+ return;
+
+ final StreamsGroupMember member = getOrMaybeCreateMember(memberId, false);
+
+ // If the commit is not transactional and the member uses the new streams protocol (KIP-1071),
+ // the member should be using the OffsetCommit API version >= 9.
+ if (!isTransactional && apiVersion < 9) {
+ throw new UnsupportedVersionException("OffsetCommit version 9 or above must be used " +
+ "by members using the streams group protocol");
+ }
+
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetFetch request.
+ *
+ * @param memberId The member id for streams groups.
+ * @param memberEpoch The member epoch for streams groups.
+ * @param lastCommittedOffset The last committed offsets in the timeline.
+ */
+ @Override
+ public void validateOffsetFetch(
+ String memberId,
+ int memberEpoch,
+ long lastCommittedOffset
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member id is null and the member epoch is -1, the request either comes
+ // from the admin client or from a client which does not provide them. In this case,
+ // the fetch request is accepted.
+ if (memberId == null && memberEpoch < 0) {
+ return;
+ }
+
+ final StreamsGroupMember member = members.get(memberId, lastCommittedOffset);
+ if (member == null) {
+ throw new UnknownMemberIdException(String.format("Member %s is not a member of group %s.",
+ memberId, groupId));
+ }
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetDelete request.
+ */
+ @Override
+ public void validateOffsetDelete() {
+ }
+
+ /**
+ * Validates the DeleteGroups request.
+ */
+ @Override
+ public void validateDeleteGroup() throws ApiException {
+ if (state() != StreamsGroupState.EMPTY) {
+ throw Errors.NON_EMPTY_GROUP.exception();
+ }
+ }
+
+ @Override
+ public boolean isSubscribedToTopic(String topic) {
+ return false;
+ }
+
+ /**
+ * Populates the list of records with tombstone(s) for deleting the group.
+ *
+ * @param records The list of records.
+ */
+ @Override
+ public void createGroupTombstoneRecords(List<CoordinatorRecord> records) {
Review Comment:
Could you please add unit tests?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
+ TopicsImage topicsImage,
+ ClusterImage clusterImage,
+ StreamsTopology topology
+ ) {
+ Set<String> requiredTopicNames = topology.requiredTopics();
+
+ // Create the topic metadata for each subscribed topic.
+ Map<String, TopicMetadata> newSubscriptionMetadata = new HashMap<>(requiredTopicNames.size());
+
+ requiredTopicNames.forEach(topicName -> {
+ TopicImage topicImage = topicsImage.getTopic(topicName);
+ if (topicImage != null) {
+ Map<Integer, Set<String>> partitionRacks = new HashMap<>();
+ topicImage.partitions().forEach((partition, partitionRegistration) -> {
+ Set<String> racks = new HashSet<>();
+ for (int replica : partitionRegistration.replicas) {
+ Optional<String> rackOptional = clusterImage.broker(replica).rack();
+ // Only add the rack if it is available for the broker/replica.
+ rackOptional.ifPresent(racks::add);
+ }
+ // If rack information is unavailable for all replicas of this partition,
+ // no corresponding entry will be stored for it in the map.
+ if (!racks.isEmpty()) {
+ partitionRacks.put(partition, racks);
+ }
+ });
+
+ newSubscriptionMetadata.put(topicName, new TopicMetadata(
+ topicImage.id(),
+ topicImage.name(),
+ topicImage.partitions().size(),
+ partitionRacks)
+ );
+ }
+ });
+
+ return Collections.unmodifiableMap(newSubscriptionMetadata);
+ }
+
+ /**
+ * Updates the metadata refresh deadline.
+ *
+ * @param deadlineMs The deadline in milliseconds.
+ * @param groupEpoch The associated group epoch.
+ */
+ public void setMetadataRefreshDeadline(
+ long deadlineMs,
+ int groupEpoch
+ ) {
+ this.metadataRefreshDeadline = new DeadlineAndEpoch(deadlineMs, groupEpoch);
+ }
+
+ /**
+ * Requests a metadata refresh.
+ */
+ public void requestMetadataRefresh() {
+ this.metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+ }
+
+ /**
+ * Checks if a metadata refresh is required. A refresh is required in two cases: 1) The deadline is smaller or equal to the current
+ * time; 2) The group epoch associated with the deadline is larger than the current group epoch. This means that the operations which
+ * updated the deadline failed.
+ *
+ * @param currentTimeMs The current time in milliseconds.
+ * @return A boolean indicating whether a refresh is required or not.
+ */
+ public boolean hasMetadataExpired(long currentTimeMs) {
+ return currentTimeMs >= metadataRefreshDeadline.deadlineMs || groupEpoch() < metadataRefreshDeadline.epoch;
+ }
+
+ /**
+ * @return The metadata refresh deadline.
+ */
+ public DeadlineAndEpoch metadataRefreshDeadline() {
+ return metadataRefreshDeadline;
+ }
+
+ /**
+ * Validates the OffsetCommit request.
+ *
+ * @param memberId The member id.
+ * @param groupInstanceId The group instance id.
+ * @param memberEpoch The member epoch.
+ * @param isTransactional Whether the offset commit is transactional or not.
+ * @param apiVersion The api version.
+ * @throws UnknownMemberIdException If the member is not found.
+ * @throws StaleMemberEpochException If the provided member epoch doesn't match the actual member epoch.
+ */
+ @Override
+ public void validateOffsetCommit(
+ String memberId,
+ String groupInstanceId,
+ int memberEpoch,
+ boolean isTransactional,
+ short apiVersion
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member epoch is -1, the request comes from either the admin client
+ // or a consumer which does not use the group management facility. In this case,
+ // the request can commit offsets if the group is empty.
+ if (memberEpoch < 0 && members().isEmpty()) return;
+
+ // The TxnOffsetCommit API does not require the member id, the generation id and the group instance id fields.
+ // Hence, they are only validated if any of them is provided
+ if (isTransactional && memberEpoch == JoinGroupRequest.UNKNOWN_GENERATION_ID &&
+ memberId.equals(JoinGroupRequest.UNKNOWN_MEMBER_ID) && groupInstanceId == null)
+ return;
+
+ final StreamsGroupMember member = getOrMaybeCreateMember(memberId, false);
+
+ // If the commit is not transactional and the member uses the new streams protocol (KIP-1071),
+ // the member should be using the OffsetCommit API version >= 9.
+ if (!isTransactional && apiVersion < 9) {
+ throw new UnsupportedVersionException("OffsetCommit version 9 or above must be used " +
+ "by members using the streams group protocol");
+ }
+
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetFetch request.
+ *
+ * @param memberId The member id for streams groups.
+ * @param memberEpoch The member epoch for streams groups.
+ * @param lastCommittedOffset The last committed offsets in the timeline.
+ */
+ @Override
+ public void validateOffsetFetch(
+ String memberId,
+ int memberEpoch,
+ long lastCommittedOffset
+ ) throws UnknownMemberIdException, StaleMemberEpochException {
+ // When the member id is null and the member epoch is -1, the request either comes
+ // from the admin client or from a client which does not provide them. In this case,
+ // the fetch request is accepted.
+ if (memberId == null && memberEpoch < 0) {
+ return;
+ }
+
+ final StreamsGroupMember member = members.get(memberId, lastCommittedOffset);
+ if (member == null) {
+ throw new UnknownMemberIdException(String.format("Member %s is not a member of group %s.",
+ memberId, groupId));
+ }
+ validateMemberEpoch(memberEpoch, member.memberEpoch());
+ }
+
+ /**
+ * Validates the OffsetDelete request.
+ */
+ @Override
+ public void validateOffsetDelete() {
+ }
+
+ /**
+ * Validates the DeleteGroups request.
+ */
+ @Override
+ public void validateDeleteGroup() throws ApiException {
+ if (state() != StreamsGroupState.EMPTY) {
+ throw Errors.NON_EMPTY_GROUP.exception();
+ }
+ }
+
+ @Override
+ public boolean isSubscribedToTopic(String topic) {
+ return false;
Review Comment:
Why constant `false`? Users cannot explicitly subscribe, but via the topology Streams subscribes to topics.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
Review Comment:
Maybe also here we might want to return an `Optional`. However, we can postpone this decision to when we will use this method in the handlers.
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
+ Map<String, TopicMetadata> partitionMetadata
+ ) {
+ this.partitionMetadata.clear();
+ this.partitionMetadata.putAll(partitionMetadata);
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Computes the partition metadata based on the current topology and the current topics/cluster image.
+ *
+ * @param topicsImage The current metadata for all available topics.
+ * @param clusterImage The current metadata for the Kafka cluster.
+ * @return An immutable map of partition metadata for each topic that the streams topology is using (besides non-repartition sink topics)
+ */
+ public Map<String, TopicMetadata> computePartitionMetadata(
Review Comment:
I assume that this will be used in the future. Could you please add unit tests?
##########
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java:
##########
@@ -0,0 +1,1180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.coordinator.group.streams;
+
+import org.apache.kafka.common.errors.ApiException;
+import org.apache.kafka.common.errors.StaleMemberEpochException;
+import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.message.ListGroupsResponseData;
+import org.apache.kafka.common.message.StreamsGroupDescribeResponseData;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.JoinGroupRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord;
+import org.apache.kafka.coordinator.group.Group;
+import org.apache.kafka.coordinator.group.OffsetExpirationCondition;
+import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl;
+import org.apache.kafka.coordinator.group.metrics.GroupCoordinatorMetricsShard;
+import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology;
+import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager;
+import org.apache.kafka.image.ClusterImage;
+import org.apache.kafka.image.TopicImage;
+import org.apache.kafka.image.TopicsImage;
+import org.apache.kafka.timeline.SnapshotRegistry;
+import org.apache.kafka.timeline.TimelineHashMap;
+import org.apache.kafka.timeline.TimelineInteger;
+import org.apache.kafka.timeline.TimelineObject;
+
+import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.ASSIGNING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.EMPTY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.NOT_READY;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.RECONCILING;
+import static org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState.STABLE;
+
+/**
+ * A Streams Group. All the metadata in this class are backed by records in the __consumer_offsets partitions.
+ */
+public class StreamsGroup implements Group {
+
+ /**
+ * The protocol type for streams groups. There is only one protocol type, "streams".
+ */
+ private static final String PROTOCOL_TYPE = "streams";
+
+ public enum StreamsGroupState {
+ EMPTY("Empty"),
+ NOT_READY("NotReady"),
+ ASSIGNING("Assigning"),
+ RECONCILING("Reconciling"),
+ STABLE("Stable"),
+ DEAD("Dead");
+
+ private final String name;
+
+ private final String lowerCaseName;
+
+ StreamsGroupState(String name) {
+ this.name = name;
+ if (Objects.equals(name, "NotReady")) {
+ this.lowerCaseName = "not_ready";
+ } else {
+ this.lowerCaseName = name.toLowerCase(Locale.ROOT);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public String toLowerCaseString() {
+ return lowerCaseName;
+ }
+ }
+
+ public static class DeadlineAndEpoch {
+
+ static final DeadlineAndEpoch EMPTY = new DeadlineAndEpoch(0L, 0);
+
+ public final long deadlineMs;
+ public final int epoch;
+
+ DeadlineAndEpoch(long deadlineMs, int epoch) {
+ this.deadlineMs = deadlineMs;
+ this.epoch = epoch;
+ }
+ }
+
+ private final LogContext logContext;
+ private final Logger log;
+
+ /**
+ * The snapshot registry.
+ */
+ private final SnapshotRegistry snapshotRegistry;
+
+ /**
+ * The group id.
+ */
+ private final String groupId;
+
+ /**
+ * The group state.
+ */
+ private final TimelineObject<StreamsGroupState> state;
+
+ /**
+ * The group epoch. The epoch is incremented whenever the topology, topic metadata or the set of members changes and it will trigger
+ * the computation of a new assignment for the group.
+ */
+ private final TimelineInteger groupEpoch;
+
+ /**
+ * The group members.
+ */
+ private final TimelineHashMap<String, StreamsGroupMember> members;
+
+ /**
+ * The static group members.
+ */
+ private final TimelineHashMap<String, String> staticMembers;
+
+ /**
+ * The metadata associated with each subscribed topic name.
+ */
+ private final TimelineHashMap<String, TopicMetadata> partitionMetadata;
+
+ /**
+ * The target assignment epoch. An assignment epoch smaller than the group epoch means that a new assignment is required. The assignment
+ * epoch is updated when a new assignment is installed.
+ */
+ private final TimelineInteger targetAssignmentEpoch;
+
+ /**
+ * The target assignment per member id.
+ */
+ private final TimelineHashMap<String, TasksTuple> targetAssignment;
+
+ /**
+ * Reverse lookup map representing tasks with their current member assignments.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetActiveTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetStandbyTasksAssignment;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetWarmupTasksAssignment;
+
+ /**
+ * These maps map each active/standby/warmup task to the process ID(s) of their current owner. When a
+ * member revokes a partition, it removes its process ID from this map. When a member gets a partition, it adds its process ID to this map.
+ */
+ private final TimelineHashMap<String, TimelineHashMap<Integer, String>> currentActiveTaskProcessId;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentStandbyTaskProcessIds;
+ private final TimelineHashMap<String, TimelineHashMap<Integer, Set<String>>> currentWarmupTaskProcessIds;
+
+ /**
+ * The coordinator metrics.
+ */
+ private final GroupCoordinatorMetricsShard metrics;
+
+ /**
+ * The Streams topology.
+ */
+ private final TimelineObject<Optional<StreamsTopology>> topology;
+
+ /**
+ * The configured topology including resolved regular expressions.
+ */
+ private final TimelineObject<Optional<ConfiguredTopology>> configuredTopology;
+
+ /**
+ * The metadata refresh deadline. It consists of a timestamp in milliseconds together with the group epoch at the time of setting it.
+ * The metadata refresh time is considered as a soft state (read that it is not stored in a timeline data structure). It is like this
+ * because it is not persisted to the log. The group epoch is here to ensure that the metadata refresh deadline is invalidated if the
+ * group epoch does not correspond to the current group epoch. This can happen if the metadata refresh deadline is updated after having
+ * refreshed the metadata but the write operation failed. In this case, the time is not automatically rolled back.
+ */
+ private DeadlineAndEpoch metadataRefreshDeadline = DeadlineAndEpoch.EMPTY;
+
+ public StreamsGroup(
+ LogContext logContext,
+ SnapshotRegistry snapshotRegistry,
+ String groupId,
+ GroupCoordinatorMetricsShard metrics
+ ) {
+ this.log = logContext.logger(StreamsGroup.class);
+ this.logContext = logContext;
+ this.snapshotRegistry = Objects.requireNonNull(snapshotRegistry);
+ this.groupId = Objects.requireNonNull(groupId);
+ this.state = new TimelineObject<>(snapshotRegistry, EMPTY);
+ this.groupEpoch = new TimelineInteger(snapshotRegistry);
+ this.members = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.staticMembers = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.partitionMetadata = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.targetAssignmentEpoch = new TimelineInteger(snapshotRegistry);
+ this.targetAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetActiveTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetStandbyTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.invertedTargetWarmupTasksAssignment = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentActiveTaskProcessId = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentStandbyTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.currentWarmupTaskProcessIds = new TimelineHashMap<>(snapshotRegistry, 0);
+ this.metrics = Objects.requireNonNull(metrics);
+ this.topology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ this.configuredTopology = new TimelineObject<>(snapshotRegistry, Optional.empty());
+ }
+
+ /**
+ * @return The group type (Streams).
+ */
+ @Override
+ public GroupType type() {
+ return GroupType.STREAMS;
+ }
+
+ /**
+ * @return The current state as a String.
+ */
+ @Override
+ public String stateAsString() {
+ return state.get().toString();
+ }
+
+ /**
+ * @return The current state as a String with given committedOffset.
+ */
+ public String stateAsString(long committedOffset) {
+ return state.get(committedOffset).toString();
+ }
+
+ /**
+ * @return the group formatted as a list group response based on the committed offset.
+ */
+ public ListGroupsResponseData.ListedGroup asListedGroup(long committedOffset) {
+ return new ListGroupsResponseData.ListedGroup()
+ .setGroupId(groupId)
+ .setProtocolType(PROTOCOL_TYPE)
+ .setGroupState(state.get(committedOffset).toString())
+ .setGroupType(type().toString());
+ }
+
+ public ConfiguredTopology configuredTopology() {
+ return configuredTopology.get().orElse(null);
+ }
+
+ public StreamsTopology topology() {
+ return topology.get().orElse(null);
+ }
+
+ public void setTopology(StreamsTopology topology) {
+ this.topology.set(Optional.of(topology));
+ maybeUpdateConfiguredTopology();
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The group id.
+ */
+ @Override
+ public String groupId() {
+ return groupId;
+ }
+
+ /**
+ * @return The current state.
+ */
+ public StreamsGroupState state() {
+ return state.get();
+ }
+
+ /**
+ * @return The current state based on committed offset.
+ */
+ public StreamsGroupState state(long committedOffset) {
+ return state.get(committedOffset);
+ }
+
+ /**
+ * @return The group epoch.
+ */
+ public int groupEpoch() {
+ return groupEpoch.get();
+ }
+
+ /**
+ * Sets the group epoch.
+ *
+ * @param groupEpoch The new group epoch.
+ */
+ public void setGroupEpoch(int groupEpoch) {
+ this.groupEpoch.set(groupEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * @return The target assignment epoch.
+ */
+ public int assignmentEpoch() {
+ return targetAssignmentEpoch.get();
+ }
+
+ /**
+ * Sets the assignment epoch.
+ *
+ * @param targetAssignmentEpoch The new assignment epoch.
+ */
+ public void setTargetAssignmentEpoch(int targetAssignmentEpoch) {
+ this.targetAssignmentEpoch.set(targetAssignmentEpoch);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Get member id of a static member that matches the given group instance id.
+ *
+ * @param groupInstanceId The group instance id.
+ * @return The member id corresponding to the given instance id or null if it does not exist
+ */
+ public String staticMemberId(String groupInstanceId) {
+ return staticMembers.get(groupInstanceId);
+ }
+
+ /**
+ * Gets or creates a new member but without adding it to the group. Adding a member is done via the
+ * {@link StreamsGroup#updateMember(StreamsGroupMember)} method.
+ *
+ * @param memberId The member id.
+ * @param createIfNotExists Booleans indicating whether the member must be created if it does not exist.
+ * @return A StreamsGroupMember.
+ */
+ public StreamsGroupMember getOrMaybeCreateMember(
+ String memberId,
+ boolean createIfNotExists
+ ) {
+ StreamsGroupMember member = members.get(memberId);
+ if (member != null) {
+ return member;
+ }
+
+ if (!createIfNotExists) {
+ throw new UnknownMemberIdException(
+ String.format("Member %s is not a member of group %s.", memberId, groupId)
+ );
+ }
+
+ return new StreamsGroupMember.Builder(memberId).build();
+ }
+
+ /**
+ * Gets a static member.
+ *
+ * @param instanceId The group instance id.
+ * @return The member corresponding to the given instance id or null if it does not exist
+ */
+ public StreamsGroupMember staticMember(String instanceId) {
+ String existingMemberId = staticMemberId(instanceId);
+ return existingMemberId == null ? null : getOrMaybeCreateMember(existingMemberId, false);
+ }
+
+ /**
+ * Adds or updates the member.
+ *
+ * @param newMember The new member state.
+ */
+ public void updateMember(StreamsGroupMember newMember) {
+ if (newMember == null) {
+ throw new IllegalArgumentException("newMember cannot be null.");
+ }
+ StreamsGroupMember oldMember = members.put(newMember.memberId(), newMember);
+ maybeUpdateTaskProcessId(oldMember, newMember);
+ updateStaticMember(newMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Updates the member id stored against the instance id if the member is a static member.
+ *
+ * @param newMember The new member state.
+ */
+ private void updateStaticMember(StreamsGroupMember newMember) {
+ if (newMember.instanceId() != null && newMember.instanceId().isPresent()) {
+ staticMembers.put(newMember.instanceId().get(), newMember.memberId());
+ }
+ }
+
+ /**
+ * Remove the member from the group.
+ *
+ * @param memberId The member id to remove.
+ */
+ public void removeMember(String memberId) {
+ StreamsGroupMember oldMember = members.remove(memberId);
+ maybeRemoveTaskProcessId(oldMember);
+ removeStaticMember(oldMember);
+ maybeUpdateGroupState();
+ }
+
+ /**
+ * Remove the static member mapping if the removed member is static.
+ *
+ * @param oldMember The member to remove.
+ */
+ private void removeStaticMember(StreamsGroupMember oldMember) {
+ if (oldMember.instanceId() != null && oldMember.instanceId().isPresent()) {
+ staticMembers.remove(oldMember.instanceId().get());
+ }
+ }
+
+ /**
+ * Returns true if the member exists.
+ *
+ * @param memberId The member id.
+ * @return A boolean indicating whether the member exists or not.
+ */
+ public boolean hasMember(String memberId) {
+ return members.containsKey(memberId);
+ }
+
+ /**
+ * @return The number of members.
+ */
+ public int numMembers() {
+ return members.size();
+ }
+
+ /**
+ * @return An immutable Map containing all the members keyed by their id.
+ */
+ public Map<String, StreamsGroupMember> members() {
+ return Collections.unmodifiableMap(members);
+ }
+
+ /**
+ * @return An immutable Map containing all the static members keyed by instance id.
+ */
+ public Map<String, String> staticMembers() {
+ return Collections.unmodifiableMap(staticMembers);
+ }
+
+ /**
+ * Returns the target assignment of the member.
+ *
+ * @return The StreamsGroupMemberAssignment or an EMPTY one if it does not exist.
+ */
+ public TasksTuple targetAssignment(String memberId) {
+ return targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY);
+ }
+
+ /**
+ * @return An immutable map containing all the topic partitions with their current member assignments.
+ */
+ public Map<String, Map<Integer, String>> invertedTargetActiveTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetActiveTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetStandbyTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetStandbyTasksAssignment);
+ }
+
+ public Map<String, Map<Integer, String>> invertedTargetWarmupTasksAssignment() {
+ return Collections.unmodifiableMap(invertedTargetWarmupTasksAssignment);
+ }
+
+ /**
+ * Updates the target assignment of a member.
+ *
+ * @param memberId The member id.
+ * @param newTargetAssignment The new target assignment.
+ */
+ public void updateTargetAssignment(String memberId, TasksTuple newTargetAssignment) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, new TasksTuple(emptyMap(), emptyMap(), emptyMap())),
+ newTargetAssignment
+ );
+ targetAssignment.put(memberId, newTargetAssignment);
+ }
+
+
+ private void updateInvertedTargetActiveTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetActiveTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetStandbyTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetStandbyTasksAssignment
+ );
+ }
+
+ private void updateInvertedTargetWarmupTasksAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment
+ ) {
+ updateInvertedTargetAssignment(
+ memberId,
+ oldTargetAssignment,
+ newTargetAssignment,
+ invertedTargetWarmupTasksAssignment
+ );
+ }
+
+ /**
+ * Updates the reverse lookup map of the target assignment.
+ *
+ * @param memberId The member Id.
+ * @param oldTargetAssignment The old target assignment.
+ * @param newTargetAssignment The new target assignment.
+ */
+ private void updateInvertedTargetAssignment(
+ String memberId,
+ TasksTuple oldTargetAssignment,
+ TasksTuple newTargetAssignment,
+ TimelineHashMap<String, TimelineHashMap<Integer, String>> invertedTargetAssignment
+ ) {
+ // Combine keys from both old and new assignments.
+ Set<String> allSubtopologyIds = new HashSet<>();
+ allSubtopologyIds.addAll(oldTargetAssignment.activeTasks().keySet());
+ allSubtopologyIds.addAll(newTargetAssignment.activeTasks().keySet());
+
+ for (String subtopologyId : allSubtopologyIds) {
+ Set<Integer> oldPartitions = oldTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+ Set<Integer> newPartitions = newTargetAssignment.activeTasks().getOrDefault(subtopologyId, Collections.emptySet());
+
+ TimelineHashMap<Integer, String> taskPartitionAssignment = invertedTargetAssignment.computeIfAbsent(
+ subtopologyId, k -> new TimelineHashMap<>(snapshotRegistry, Math.max(oldPartitions.size(), newPartitions.size()))
+ );
+
+ // Remove partitions that aren't present in the new assignment only if the partition is currently
+ // still assigned to the member in question.
+ // If p0 was moved from A to B, and the target assignment map was updated for B first, we don't want to
+ // remove the key p0 from the inverted map and undo the action when A eventually tries to update its assignment.
+ for (Integer partition : oldPartitions) {
+ if (!newPartitions.contains(partition) && memberId.equals(taskPartitionAssignment.get(partition))) {
+ taskPartitionAssignment.remove(partition);
+ }
+ }
+
+ // Add partitions that are in the new assignment but not in the old assignment.
+ for (Integer partition : newPartitions) {
+ if (!oldPartitions.contains(partition)) {
+ taskPartitionAssignment.put(partition, memberId);
+ }
+ }
+
+ if (taskPartitionAssignment.isEmpty()) {
+ invertedTargetAssignment.remove(subtopologyId);
+ } else {
+ invertedTargetAssignment.put(subtopologyId, taskPartitionAssignment);
+ }
+ }
+ }
+
+ /**
+ * Removes the target assignment of a member.
+ *
+ * @param memberId The member id.
+ */
+ public void removeTargetAssignment(String memberId) {
+ updateInvertedTargetActiveTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetStandbyTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ updateInvertedTargetWarmupTasksAssignment(
+ memberId,
+ targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY),
+ TasksTuple.EMPTY
+ );
+ targetAssignment.remove(memberId);
+ }
+
+ /**
+ * @return An immutable Map containing all the target assignment keyed by member id.
+ */
+ public Map<String, TasksTuple> targetAssignment() {
+ return Collections.unmodifiableMap(targetAssignment);
+ }
+
+ /**
+ * Returns the current processId of a task or null if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The processId or null.
+ */
+ public String currentActiveTaskProcessId(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, String> tasks = currentActiveTaskProcessId.get(subtopologyId);
+ if (tasks == null) {
+ return null;
+ } else {
+ return tasks.getOrDefault(taskId, null);
+ }
+ }
+
+ /**
+ * Returns the current member IDs of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentStandbyTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentStandbyTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * Returns the current member ID of a task or empty set if the task does not have one.
+ *
+ * @param subtopologyId The topic id.
+ * @param taskId The task id.
+ * @return The member IDs or empty set.
+ */
+ public Set<String> currentWarmupTaskProcessIds(
+ String subtopologyId, int taskId
+ ) {
+ Map<Integer, Set<String>> tasks = currentWarmupTaskProcessIds.get(subtopologyId);
+ if (tasks == null) {
+ return Collections.emptySet();
+ } else {
+ return tasks.getOrDefault(taskId, Collections.emptySet());
+ }
+ }
+
+ /**
+ * @return An immutable Map of partition metadata for each topic that are inputs for this streams group.
+ */
+ public Map<String, TopicMetadata> partitionMetadata() {
+ return Collections.unmodifiableMap(partitionMetadata);
+ }
+
+ /**
+ * Updates the partition metadata. This replaces the previous one.
+ *
+ * @param partitionMetadata The new partition metadata.
+ */
+ public void setPartitionMetadata(
Review Comment:
Could you add unit test for this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org