You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2022/04/25 03:56:34 UTC
[GitHub] [kafka] showuon commented on a diff in pull request #11983: KAFKA-13763 (2): Refactor IncrementalCooperativeAssignor for improved unit testing
showuon commented on code in PR #11983:
URL: https://github.com/apache/kafka/pull/11983#discussion_r857235521
##########
connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java:
##########
@@ -1142,107 +982,65 @@ private void removeConnector(String connector) {
);
}
- private void updateConfigSnapshot() {
- when(coordinator.configSnapshot()).thenReturn(configState());
- }
-
private ClusterConfigState configState() {
Map<String, Integer> taskCounts = new HashMap<>(connectors);
- Map<String, Map<String, String>> connectorConfigs = taskCounts.keySet().stream().collect(Collectors.toMap(
- Function.identity(),
- connector -> Collections.emptyMap()
- ));
- Map<String, TargetState> targetStates = taskCounts.keySet().stream().collect(Collectors.toMap(
- Function.identity(),
- connector -> TargetState.STARTED
- ));
+ Map<String, Map<String, String>> connectorConfigs = transformValues(taskCounts, c -> Collections.emptyMap());
+ Map<String, TargetState> targetStates = transformValues(taskCounts, c -> TargetState.STARTED);
Map<ConnectorTaskId, Map<String, String>> taskConfigs = taskCounts.entrySet().stream()
.flatMap(e -> IntStream.range(0, e.getValue()).mapToObj(i -> new ConnectorTaskId(e.getKey(), i)))
.collect(Collectors.toMap(
Function.identity(),
connectorTaskId -> Collections.emptyMap()
));
return new ClusterConfigState(
- offset,
+ 16,
Review Comment:
Could you explain the `16` here? Thanks.
##########
connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java:
##########
@@ -44,93 +35,49 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import static org.apache.kafka.connect.runtime.distributed.ExtendedAssignment.duplicate;
-import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V1;
-import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeConnectProtocol.CONNECT_PROTOCOL_V2;
+import static org.apache.kafka.connect.runtime.distributed.IncrementalCooperativeAssignor.ClusterAssignment;
import static org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad;
+import static org.apache.kafka.connect.util.ConnectUtils.transformValues;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.runners.Parameterized.Parameter;
-import static org.junit.runners.Parameterized.Parameters;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-@RunWith(Parameterized.class)
-public class IncrementalCooperativeAssignorTest {
- @Rule
- public MockitoRule rule = MockitoJUnit.rule();
-
- @Mock
- private WorkerCoordinator coordinator;
-
- @Captor
- ArgumentCaptor<Map<String, ExtendedAssignment>> assignmentsCapture;
-
- @Parameters
- public static Iterable<?> mode() {
- return Arrays.asList(CONNECT_PROTOCOL_V1, CONNECT_PROTOCOL_V2);
- }
Review Comment:
So, we don't need to test these 2 modes now?
##########
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java:
##########
@@ -667,17 +706,12 @@ private static <T> Map<String, Collection<T>> diff(Map<String, Collection<T>> ba
return incremental;
}
- private ConnectorsAndTasks assignment(Map<String, ExtendedWorkerState> memberConfigs) {
- log.debug("Received assignments: {}", memberConfigs);
- Set<String> connectors = memberConfigs.values()
- .stream()
- .flatMap(state -> state.assignment().connectors().stream())
- .collect(Collectors.toSet());
- Set<ConnectorTaskId> tasks = memberConfigs.values()
- .stream()
- .flatMap(state -> state.assignment().tasks().stream())
- .collect(Collectors.toSet());
- return new ConnectorsAndTasks.Builder().with(connectors, tasks).build();
+ private ConnectorsAndTasks assignment(Map<String, ConnectorsAndTasks> memberAssignments) {
+ log.debug("Received assignments: {}", memberAssignments);
+ return new ConnectorsAndTasks.Builder().with(
+ ConnectUtils.combineCollections(memberAssignments.values(), ConnectorsAndTasks::connectors),
+ ConnectUtils.combineCollections(memberAssignments.values(), ConnectorsAndTasks::tasks)
Review Comment:
Nice cleanup
##########
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java:
##########
@@ -298,20 +329,21 @@ protected Map<String, ByteBuffer> performTaskAssignment(String leaderId, long ma
Map<String, Collection<ConnectorTaskId>> incrementalTaskAssignments =
diff(taskAssignments, currentTaskAssignments);
+ previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments);
+ previousGenerationId = currentGenerationId;
+ previousMembers = memberAssignments.keySet();
+
log.debug("Incremental connector assignments: {}", incrementalConnectorAssignments);
log.debug("Incremental task assignments: {}", incrementalTaskAssignments);
- coordinator.leaderState(new LeaderState(memberConfigs, connectorAssignments, taskAssignments));
Review Comment:
I'm not 100% confident that the `connectorAssignments` == `clusterAssignment.allAssignedConnectors()`, and `taskAssignments` == `clusterAssignment.allAssignedTasks()`. Could you help me explain and confirm it?
##########
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java:
##########
@@ -745,22 +779,108 @@ protected void assignTasks(List<WorkerLoad> workerAssignment, Collection<Connect
}
}
- private static List<WorkerLoad> workerAssignment(Map<String, ExtendedWorkerState> memberConfigs,
+ private static List<WorkerLoad> workerAssignment(Map<String, ConnectorsAndTasks> memberAssignments,
ConnectorsAndTasks toExclude) {
ConnectorsAndTasks ignore = new ConnectorsAndTasks.Builder()
.with(new HashSet<>(toExclude.connectors()), new HashSet<>(toExclude.tasks()))
.build();
- return memberConfigs.entrySet().stream()
+ return memberAssignments.entrySet().stream()
.map(e -> new WorkerLoad.Builder(e.getKey()).with(
- e.getValue().assignment().connectors().stream()
+ e.getValue().connectors().stream()
.filter(v -> !ignore.connectors().contains(v))
.collect(Collectors.toList()),
- e.getValue().assignment().tasks().stream()
+ e.getValue().tasks().stream()
.filter(v -> !ignore.tasks().contains(v))
.collect(Collectors.toList())
).build()
).collect(Collectors.toList());
}
+ static class ClusterAssignment {
Review Comment:
I think we need to override `toString` for `ClusterAssignment`. WDYT?
##########
connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java:
##########
@@ -344,9 +376,9 @@ private ConnectorsAndTasks computePreviousAssignment(Map<String, ConnectorsAndTa
Map<String, Collection<ConnectorTaskId>> taskAssignments,
ConnectorsAndTasks lostAssignments) {
ConnectorsAndTasks previousAssignment = new ConnectorsAndTasks.Builder().with(
- connectorAssignments.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()),
- taskAssignments.values() .stream() .flatMap(Collection::stream).collect(Collectors.toSet()))
- .build();
+ ConnectUtils.combineCollections(connectorAssignments.values()),
+ ConnectUtils.combineCollections(taskAssignments.values())
Review Comment:
nice cleanup
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org