You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2022/07/12 07:13:50 UTC

[GitHub] [kafka] showuon commented on a diff in pull request #11783: KAFKA-10000: System tests (KIP-618)

showuon commented on code in PR #11783:
URL: https://github.com/apache/kafka/pull/11783#discussion_r918618483


##########
tests/kafkatest/tests/connect/connect_distributed_test.py:
##########
@@ -519,6 +535,105 @@ def test_bounce(self, clean, connect_protocol):
 
         assert success, "Found validation errors:\n" + "\n  ".join(errors)
 
+    @cluster(num_nodes=6)
+    @matrix(clean=[True, False], connect_protocol=['sessioned', 'compatible', 'eager'])
+    def test_exactly_once_source(self, clean, connect_protocol):
+        """
+        Validates that source and sink tasks that run continuously and produce a predictable sequence of messages

Review Comment:
   1. Did we validate `sink` tasks in the test?
   2. Should we mention "exactly once" here?



##########
tests/kafkatest/tests/connect/connect_distributed_test.py:
##########
@@ -519,6 +535,105 @@ def test_bounce(self, clean, connect_protocol):
 
         assert success, "Found validation errors:\n" + "\n  ".join(errors)
 
+    @cluster(num_nodes=6)
+    @matrix(clean=[True, False], connect_protocol=['sessioned', 'compatible', 'eager'])
+    def test_exactly_once_source(self, clean, connect_protocol):
+        """
+        Validates that source and sink tasks that run continuously and produce a predictable sequence of messages
+        run correctly and deliver messages exactly once when Kafka Connect workers undergo bounces, both clean and unclean.
+        """
+        num_tasks = 3
+
+        self.EXACTLY_ONCE_SOURCE_SUPPORT = 'enabled'
+        self.CONNECT_PROTOCOL = connect_protocol
+        self.setup_services()
+        self.cc.set_configs(lambda node: self.render("connect-distributed.properties", node=node))
+        self.cc.start()
+
+        self.source = VerifiableSource(self.cc, topic=self.TOPIC, tasks=num_tasks, throughput=100, complete_records=True)
+        self.source.start()
+
+        for i in range(4):
+            start = i % len(self.cc.nodes)
+            # Don't want to restart worker nodes in the same order every time
+            shuffled_nodes = self.cc.nodes[start:] + self.cc.nodes[:start]
+            for node in shuffled_nodes:
+                started = time.time()
+                self.logger.info("%s bouncing Kafka Connect on %s", clean and "Clean" or "Hard", str(node.account))
+                self.cc.stop_node(node, clean_shutdown=clean)
+                with node.account.monitor_log(self.cc.LOG_FILE) as monitor:
+                    self.cc.start_node(node)
+                    monitor.wait_until("Starting connectors and tasks using config offset", timeout_sec=90,
+                                       err_msg="Kafka Connect worker didn't successfully join group and start work")
+                self.logger.info("Bounced Kafka Connect on %s and rejoined in %f seconds", node.account, time.time() - started)
+
+                if i >= 2:
+                    # Give additional time for the worker group to recover. Even if it is not a hard bounce, there are
+                    # some cases where a restart can cause a rebalance to take the full length of the session timeout
+                    # (e.g. if the client shuts down before it has received the memberId from its initial JoinGroup).
+                    # If we don't give enough time for the group to stabilize, the next bounce may cause workers to
+                    # be shut down before they have any time to process data and we can end up with zero data making it
+                    # through the test.
+                    time.sleep(15)
+                else:
+                    # We also need to make sure that, even without time for the cluster to recover gracefully in between
+                    # worker restarts, the cluster and its tasks do not get into an inconsistent state and either duplicate or
+                    # drop messages.
+                    pass

Review Comment:
   Nice test!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org