[kafka] branch trunk updated: MINOR: Disable those flaky tests (#11895)

[gu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

guozhang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new cad4985  MINOR: Disable those flaky tests (#11895)
cad4985 is described below

commit cad4985a0a15c6372e9b42260c2468d18d93e3d6
Author: Guozhang Wang <wa...@gmail.com>
AuthorDate: Mon Mar 14 21:32:28 2022 -0700

    MINOR: Disable those flaky tests (#11895)
    
    I collected a list of the most flaky tests observed lately, checked / created their corresponding tickets, and mark them as ignored for now. Many of these failures are:
    
    0. Failing very frequently in the past (at least in my observations).
    1. not investigated for some time.
    2. have a PR for review (mostly thanks to @showuon !), but not reviewed for some time.
    
    Since 0), these tests failures are hindering our development; and from 1/2) above, people are either too busy to look after them, or honestly the tests are not considered as providing values since otherwise people should care enough to panic and try to resolve. So I think it's reasonable to disable all these tests for now. If we later learned our lesson a hard way, it would motivate us to tackle flaky tests more diligently as well.
    
    I'm only disabling those tests that have been failed for a while, and if for such time no one have been looking into them, I'm concerned that just gossiping around about those flakiness would not bring people's attention to them either. So my psychological motivation is that "if people do not care about those failed tests for weeks (which, is not a good thing! :P), let's teach ourselves the lesson a hard way when it indeed buries a bug that bites us, or not learn the lesson at all --- [...]
    
    Reviewers: John Roesler <vv...@apache.org>, Matthias J. Sax <mj...@apache.org>, Luke Chen <sh...@gmail.com>, Randall Hauch <rh...@gmail.com>
---
 .../integration/RebalanceSourceConnectorsIntegrationTest.java        | 4 ++--
 core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala   | 2 ++
 .../integration/kafka/server/DynamicBrokerReconfigurationTest.scala  | 4 ++++
 core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala | 5 ++++-
 core/src/test/scala/unit/kafka/network/SocketServerTest.scala        | 2 ++
 5 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
index ae3e20d..855882c 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java
@@ -188,6 +188,7 @@ public class RebalanceSourceConnectorsIntegrationTest {
     }
 
     @Test
+    @Ignore // TODO: To be re-enabled once we can make it less flaky (KAFKA-8391)
     public void testDeleteConnector() throws Exception {
         // create test topic
         connect.kafka().createTopic(TOPIC_NAME, NUM_TOPIC_PARTITIONS);
@@ -269,8 +270,7 @@ public class RebalanceSourceConnectorsIntegrationTest {
                 WORKER_SETUP_DURATION_MS, "Connect and tasks are imbalanced between the workers.");
     }
 
-    // should enable it after KAFKA-12495 fixed
-    @Ignore
+    @Ignore // TODO: To be re-enabled once we can make it less flaky (KAFKA-12495, KAFKA-12283)
     @Test
     public void testMultipleWorkersRejoining() throws Exception {
         // create test topic
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index f675ef5..b3afc50 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -211,6 +211,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
   }
 
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky (KAFKA-7540)
   def testClose(): Unit = {
     val numRecords = 10
     val producer = createProducer()
@@ -299,6 +300,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     * Then, 1 consumer should be left out of the group.
     */
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky (KAFKA-13421)
   def testRollingBrokerRestartsWithSmallerMaxGroupSizeConfigDisruptsBigGroup(): Unit = {
     val group = "group-max-size-test"
     val topic = "group-max-size-test"
diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
index 0cc5862..1d64106 100644
--- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala
@@ -552,6 +552,7 @@ class DynamicBrokerReconfigurationTest extends QuorumTestHarness with SaslSetup
   }
 
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky: KAFKA-6527
   def testDefaultTopicConfig(): Unit = {
     val (producerThread, consumerThread) = startProduceConsume(retries = 0)
 
@@ -665,6 +666,7 @@ class DynamicBrokerReconfigurationTest extends QuorumTestHarness with SaslSetup
   }
 
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky: KAFKA-8280
   def testUncleanLeaderElectionEnable(): Unit = {
     val controller = servers.find(_.config.brokerId == TestUtils.waitUntilControllerElected(zkClient)).get
     val controllerId = controller.config.brokerId
@@ -726,6 +728,7 @@ class DynamicBrokerReconfigurationTest extends QuorumTestHarness with SaslSetup
   }
 
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky (KAFKA-13672)
   def testThreadPoolResize(): Unit = {
     val requestHandlerPrefix = "data-plane-kafka-request-handler-"
     val networkThreadPrefix = "data-plane-kafka-network-thread-"
@@ -864,6 +867,7 @@ class DynamicBrokerReconfigurationTest extends QuorumTestHarness with SaslSetup
   }
 
   @Test
+  @Disabled // TODO: To be re-enabled once we can make it less flaky (KAFKA-7957)
   def testMetricsReporterUpdate(): Unit = {
     // Add a new metrics reporter
     val newProps = new Properties
diff --git a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
index 4a2a401..e9efdab 100644
--- a/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
+++ b/core/src/test/scala/unit/kafka/admin/LeaderElectionCommandTest.scala
@@ -32,7 +32,7 @@ import org.apache.kafka.common.TopicPartition
 import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
 import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.extension.ExtendWith
-import org.junit.jupiter.api.{BeforeEach, Tag}
+import org.junit.jupiter.api.{BeforeEach, Disabled, Tag}
 
 @ExtendWith(value = Array(classOf[ClusterTestExtensions]))
 @ClusterTestDefaults(clusterType = Type.BOTH, brokers = 3)
@@ -86,6 +86,7 @@ final class LeaderElectionCommandTest(cluster: ClusterInstance) {
   }
 
   @ClusterTest
+  @Disabled // TODO: re-enable until we fixed KAFKA-8541
   def testTopicPartition(): Unit = {
     val client = cluster.createAdminClient()
     val topic = "unclean-topic"
@@ -119,6 +120,7 @@ final class LeaderElectionCommandTest(cluster: ClusterInstance) {
   }
 
   @ClusterTest
+  @Disabled // TODO: re-enable until we fixed KAFKA-8785
   def testPathToJsonFile(): Unit = {
     val client = cluster.createAdminClient()
     val topic = "unclean-topic"
@@ -153,6 +155,7 @@ final class LeaderElectionCommandTest(cluster: ClusterInstance) {
   }
 
   @ClusterTest
+  @Disabled // TODO: re-enable after KAFKA-13737 is fixed
   def testPreferredReplicaElection(): Unit = {
     val client = cluster.createAdminClient()
     val topic = "preferred-topic"
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index 849646c..33d15ad1 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -1389,6 +1389,7 @@ class SocketServerTest {
    * buffered receive.
    */
   @Test
+  @Disabled // TODO: re-enabled until KAFKA-13735 is fixed
   def remoteCloseWithoutBufferedReceives(): Unit = {
     verifyRemoteCloseWithBufferedReceives(numComplete = 0, hasIncomplete = false)
   }
@@ -1426,6 +1427,7 @@ class SocketServerTest {
    * The channel must be closed after pending receives are processed.
    */
   @Test
+  @Disabled // TODO: re-enable after KAFKA-13736 is fixed
   def closingChannelWithBufferedReceives(): Unit = {
     verifyRemoteCloseWithBufferedReceives(numComplete = 3, hasIncomplete = false, makeClosing = true)
   }