You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2024/05/03 06:25:06 UTC

(kafka) branch trunk updated: KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)

This is an automated email from the ASF dual-hosted git repository.

chia7712 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 87390f961f3 KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)
87390f961f3 is described below

commit 87390f961f3bd0c582d0926cb9c4fe00d9298b6f
Author: PoAn Yang <pa...@apache.org>
AuthorDate: Fri May 3 14:24:59 2024 +0800

    KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)
    
    Reviewers: Luke Chen <sh...@gmail.com>, Chia-Ping Tsai <ch...@gmail.com>
---
 core/src/test/java/kafka/test/ClusterConfig.java   | 54 +++++++----------
 .../test/java/kafka/test/ClusterConfigTest.java    | 32 ++++++++--
 .../java/kafka/test/ClusterTestExtensionsTest.java | 20 +++++++
 .../java/kafka/test/annotation/ClusterTest.java    |  1 +
 .../kafka/test/annotation/ClusterTestDefaults.java |  1 +
 .../kafka/test/junit/ClusterTestExtensions.java    | 70 +++++-----------------
 .../test/junit/RaftClusterInvocationContext.java   |  1 +
 .../test/junit/ZkClusterInvocationContext.java     |  5 ++
 8 files changed, 92 insertions(+), 92 deletions(-)

diff --git a/core/src/test/java/kafka/test/ClusterConfig.java b/core/src/test/java/kafka/test/ClusterConfig.java
index 0bb1ddad481..4a713f35928 100644
--- a/core/src/test/java/kafka/test/ClusterConfig.java
+++ b/core/src/test/java/kafka/test/ClusterConfig.java
@@ -38,6 +38,7 @@ public class ClusterConfig {
     private final Type type;
     private final int brokers;
     private final int controllers;
+    private final int disksPerBroker;
     private final String name;
     private final boolean autoStart;
 
@@ -55,14 +56,20 @@ public class ClusterConfig {
     private final Map<Integer, Map<String, String>> perBrokerOverrideProperties;
 
     @SuppressWarnings("checkstyle:ParameterNumber")
-    private ClusterConfig(Type type, int brokers, int controllers, String name, boolean autoStart,
+    private ClusterConfig(Type type, int brokers, int controllers, int disksPerBroker, String name, boolean autoStart,
                   SecurityProtocol securityProtocol, String listenerName, File trustStoreFile,
                   MetadataVersion metadataVersion, Map<String, String> serverProperties, Map<String, String> producerProperties,
                   Map<String, String> consumerProperties, Map<String, String> adminClientProperties, Map<String, String> saslServerProperties,
                   Map<String, String> saslClientProperties, Map<Integer, Map<String, String>> perBrokerOverrideProperties) {
+        // do fail fast. the following values are invalid for both zk and kraft modes.
+        if (brokers < 0) throw new IllegalArgumentException("Number of brokers must be greater or equal to zero.");
+        if (controllers < 0) throw new IllegalArgumentException("Number of controller must be greater or equal to zero.");
+        if (disksPerBroker <= 0) throw new IllegalArgumentException("Number of disks must be greater than zero.");
+
         this.type = Objects.requireNonNull(type);
         this.brokers = brokers;
         this.controllers = controllers;
+        this.disksPerBroker = disksPerBroker;
         this.name = name;
         this.autoStart = autoStart;
         this.securityProtocol = Objects.requireNonNull(securityProtocol);
@@ -90,6 +97,10 @@ public class ClusterConfig {
         return controllers;
     }
 
+    public int numDisksPerBroker() {
+        return disksPerBroker;
+    }
+
     public Optional<String> name() {
         return Optional.ofNullable(name);
     }
@@ -151,42 +162,12 @@ public class ClusterConfig {
         return tags;
     }
 
-    @SuppressWarnings({"CyclomaticComplexity"})
-    @Override
-    public boolean equals(Object object) {
-        if (this == object) return true;
-        if (object == null || getClass() != object.getClass()) return false;
-        ClusterConfig clusterConfig = (ClusterConfig) object;
-        return Objects.equals(type, clusterConfig.type)
-                && Objects.equals(brokers, clusterConfig.brokers)
-                && Objects.equals(controllers, clusterConfig.controllers)
-                && Objects.equals(name, clusterConfig.name)
-                && Objects.equals(autoStart, clusterConfig.autoStart)
-                && Objects.equals(securityProtocol, clusterConfig.securityProtocol)
-                && Objects.equals(listenerName, clusterConfig.listenerName)
-                && Objects.equals(trustStoreFile, clusterConfig.trustStoreFile)
-                && Objects.equals(metadataVersion, clusterConfig.metadataVersion)
-                && Objects.equals(serverProperties, clusterConfig.serverProperties)
-                && Objects.equals(producerProperties, clusterConfig.producerProperties)
-                && Objects.equals(consumerProperties, clusterConfig.consumerProperties)
-                && Objects.equals(adminClientProperties, clusterConfig.adminClientProperties)
-                && Objects.equals(saslServerProperties, clusterConfig.saslServerProperties)
-                && Objects.equals(saslClientProperties, clusterConfig.saslClientProperties)
-                && Objects.equals(perBrokerOverrideProperties, clusterConfig.perBrokerOverrideProperties);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(type, brokers, controllers, name, autoStart, securityProtocol, listenerName,
-                trustStoreFile, metadataVersion, serverProperties, producerProperties, consumerProperties,
-                adminClientProperties, saslServerProperties, saslClientProperties, perBrokerOverrideProperties);
-    }
-
     public static Builder defaultBuilder() {
         return new Builder()
                 .setType(Type.ZK)
                 .setBrokers(1)
                 .setControllers(1)
+                .setDisksPerBroker(1)
                 .setAutoStart(true)
                 .setSecurityProtocol(SecurityProtocol.PLAINTEXT)
                 .setMetadataVersion(MetadataVersion.latestTesting());
@@ -201,6 +182,7 @@ public class ClusterConfig {
                 .setType(clusterConfig.type)
                 .setBrokers(clusterConfig.brokers)
                 .setControllers(clusterConfig.controllers)
+                .setDisksPerBroker(clusterConfig.disksPerBroker)
                 .setName(clusterConfig.name)
                 .setAutoStart(clusterConfig.autoStart)
                 .setSecurityProtocol(clusterConfig.securityProtocol)
@@ -220,6 +202,7 @@ public class ClusterConfig {
         private Type type;
         private int brokers;
         private int controllers;
+        private int disksPerBroker;
         private String name;
         private boolean autoStart;
         private SecurityProtocol securityProtocol;
@@ -251,6 +234,11 @@ public class ClusterConfig {
             return this;
         }
 
+        public Builder setDisksPerBroker(int disksPerBroker) {
+            this.disksPerBroker = disksPerBroker;
+            return this;
+        }
+
         public Builder setName(String name) {
             this.name = name;
             return this;
@@ -319,7 +307,7 @@ public class ClusterConfig {
         }
 
         public ClusterConfig build() {
-            return new ClusterConfig(type, brokers, controllers, name, autoStart, securityProtocol, listenerName,
+            return new ClusterConfig(type, brokers, controllers, disksPerBroker, name, autoStart, securityProtocol, listenerName,
                     trustStoreFile, metadataVersion, serverProperties, producerProperties, consumerProperties,
                     adminClientProperties, saslServerProperties, saslClientProperties,
                     perBrokerOverrideProperties);
diff --git a/core/src/test/java/kafka/test/ClusterConfigTest.java b/core/src/test/java/kafka/test/ClusterConfigTest.java
index 20f9e8f6c33..790ca7d0e5e 100644
--- a/core/src/test/java/kafka/test/ClusterConfigTest.java
+++ b/core/src/test/java/kafka/test/ClusterConfigTest.java
@@ -49,6 +49,7 @@ public class ClusterConfigTest {
                 .setType(Type.KRAFT)
                 .setBrokers(3)
                 .setControllers(2)
+                .setDisksPerBroker(1)
                 .setName("builder-test")
                 .setAutoStart(true)
                 .setSecurityProtocol(SecurityProtocol.PLAINTEXT)
@@ -64,12 +65,35 @@ public class ClusterConfigTest {
                 .setPerBrokerProperties(Collections.singletonMap(0, Collections.singletonMap("broker_0", "broker_0_value")))
                 .build();
 
-        ClusterConfig copy = ClusterConfig.builder(clusterConfig).build();
-        Assertions.assertEquals(clusterConfig, copy);
-        Assertions.assertEquals(clusterConfig.hashCode(), copy.hashCode());
-
         Map<String, Object> clusterConfigFields = fields(clusterConfig);
         Map<String, Object> copyFields = fields(clusterConfig);
         Assertions.assertEquals(clusterConfigFields, copyFields);
     }
+
+    @Test
+    public void testBrokerLessThanZero() {
+        Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+                .setBrokers(-1)
+                .setControllers(1)
+                .setDisksPerBroker(1)
+                .build());
+    }
+
+    @Test
+    public void testControllersLessThanZero() {
+        Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+                .setBrokers(1)
+                .setControllers(-1)
+                .setDisksPerBroker(1)
+                .build());
+    }
+
+    @Test
+    public void testDisksPerBrokerIsZero() {
+        Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+                .setBrokers(1)
+                .setControllers(1)
+                .setDisksPerBroker(0)
+                .build());
+    }
 }
diff --git a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
index ada1d1b21b4..0eb94abd660 100644
--- a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
+++ b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
@@ -25,12 +25,15 @@ import kafka.test.annotation.ClusterTestDefaults;
 import kafka.test.annotation.ClusterTests;
 import kafka.test.annotation.Type;
 import kafka.test.junit.ClusterTestExtensions;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.DescribeLogDirsResult;
 import org.apache.kafka.server.common.MetadataVersion;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 
 @ClusterTestDefaults(clusterType = Type.ZK, serverProperties = {
@@ -104,6 +107,23 @@ public class ClusterTestExtensionsTest {
         }
     }
 
+    @ClusterTests({
+        @ClusterTest(clusterType = Type.ZK),
+        @ClusterTest(clusterType = Type.ZK, disksPerBroker = 2),
+        @ClusterTest(clusterType = Type.KRAFT),
+        @ClusterTest(clusterType = Type.KRAFT, disksPerBroker = 2),
+        @ClusterTest(clusterType = Type.CO_KRAFT),
+        @ClusterTest(clusterType = Type.CO_KRAFT, disksPerBroker = 2)
+    })
+    public void testClusterTestWithDisksPerBroker() throws ExecutionException, InterruptedException {
+        Admin admin = clusterInstance.createAdminClient();
+
+        DescribeLogDirsResult result = admin.describeLogDirs(clusterInstance.brokerIds());
+        result.allDescriptions().get().forEach((brokerId, logDirDescriptionMap) -> {
+            Assertions.assertEquals(clusterInstance.config().numDisksPerBroker(), logDirDescriptionMap.size());
+        });
+    }
+
     @ClusterTest(autoStart = AutoStart.NO)
     public void testNoAutoStart() {
         Assertions.assertThrows(RuntimeException.class, clusterInstance::anyBrokerSocketServer);
diff --git a/core/src/test/java/kafka/test/annotation/ClusterTest.java b/core/src/test/java/kafka/test/annotation/ClusterTest.java
index 8880966dea8..c3c9364acd0 100644
--- a/core/src/test/java/kafka/test/annotation/ClusterTest.java
+++ b/core/src/test/java/kafka/test/annotation/ClusterTest.java
@@ -36,6 +36,7 @@ public @interface ClusterTest {
     Type clusterType() default Type.DEFAULT;
     int brokers() default 0;
     int controllers() default 0;
+    int disksPerBroker() default 0;
     AutoStart autoStart() default AutoStart.DEFAULT;
 
     String name() default "";
diff --git a/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java b/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
index 2f9efd777ef..af45b877652 100644
--- a/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
+++ b/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
@@ -38,6 +38,7 @@ public @interface ClusterTestDefaults {
     Type clusterType() default Type.ZK;
     int brokers() default 1;
     int controllers() default 1;
+    int disksPerBroker() default 1;
     boolean autoStart() default true;
     // Set default server properties for all @ClusterTest(s)
     ClusterConfigProperty[] serverProperties() default {};
diff --git a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
index 5a5d78e989c..bdf3b2b9522 100644
--- a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
+++ b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
@@ -19,6 +19,7 @@ package kafka.test.junit;
 
 import kafka.test.ClusterConfig;
 import kafka.test.ClusterGenerator;
+import kafka.test.annotation.AutoStart;
 import kafka.test.annotation.ClusterTestDefaults;
 import kafka.test.annotation.ClusterConfigProperty;
 import kafka.test.annotation.ClusterTemplate;
@@ -139,59 +140,7 @@ public class ClusterTestExtensions implements TestTemplateInvocationContextProvi
 
     private void processClusterTest(ExtensionContext context, ClusterTest annot, ClusterTestDefaults defaults,
                                     Consumer<TestTemplateInvocationContext> testInvocations) {
-        final Type type;
-        if (annot.clusterType() == Type.DEFAULT) {
-            type = defaults.clusterType();
-        } else {
-            type = annot.clusterType();
-        }
-
-        final int brokers;
-        if (annot.brokers() == 0) {
-            brokers = defaults.brokers();
-        } else {
-            brokers = annot.brokers();
-        }
-
-        final int controllers;
-        if (annot.controllers() == 0) {
-            controllers = defaults.controllers();
-        } else {
-            controllers = annot.controllers();
-        }
-
-        if (brokers <= 0 || controllers <= 0) {
-            throw new IllegalArgumentException("Number of brokers/controllers must be greater than zero.");
-        }
-
-        final boolean autoStart;
-        switch (annot.autoStart()) {
-            case YES:
-                autoStart = true;
-                break;
-            case NO:
-                autoStart = false;
-                break;
-            case DEFAULT:
-                autoStart = defaults.autoStart();
-                break;
-            default:
-                throw new IllegalStateException();
-        }
-
-        ClusterConfig.Builder configBuilder = ClusterConfig.builder()
-                .setType(type)
-                .setBrokers(brokers)
-                .setControllers(controllers)
-                .setAutoStart(autoStart)
-                .setSecurityProtocol(annot.securityProtocol())
-                .setMetadataVersion(annot.metadataVersion());
-        if (!annot.name().isEmpty()) {
-            configBuilder.setName(annot.name());
-        }
-        if (!annot.listener().isEmpty()) {
-            configBuilder.setListenerName(annot.listener());
-        }
+        Type type = annot.clusterType() == Type.DEFAULT ? defaults.clusterType() : annot.clusterType();
 
         Map<String, String> serverProperties = new HashMap<>();
         for (ClusterConfigProperty property : defaults.serverProperties()) {
@@ -200,8 +149,19 @@ public class ClusterTestExtensions implements TestTemplateInvocationContextProvi
         for (ClusterConfigProperty property : annot.serverProperties()) {
             serverProperties.put(property.key(), property.value());
         }
-        configBuilder.setServerProperties(serverProperties);
-        type.invocationContexts(context.getRequiredTestMethod().getName(), configBuilder.build(), testInvocations);
+        ClusterConfig config = ClusterConfig.builder()
+                .setType(type)
+                .setBrokers(annot.brokers() == 0 ? defaults.brokers() : annot.brokers())
+                .setControllers(annot.controllers() == 0 ? defaults.controllers() : annot.controllers())
+                .setDisksPerBroker(annot.disksPerBroker() == 0 ? defaults.disksPerBroker() : annot.disksPerBroker())
+                .setAutoStart(annot.autoStart() == AutoStart.DEFAULT ? defaults.autoStart() : annot.autoStart() == AutoStart.YES)
+                .setName(annot.name().trim().isEmpty() ? null : annot.name())
+                .setListenerName(annot.listener().trim().isEmpty() ? null : annot.listener())
+                .setServerProperties(serverProperties)
+                .setSecurityProtocol(annot.securityProtocol())
+                .setMetadataVersion(annot.metadataVersion())
+                .build();
+        type.invocationContexts(context.getRequiredTestMethod().getName(), config, testInvocations);
     }
 
     private ClusterTestDefaults getClusterTestDefaults(Class<?> testClass) {
diff --git a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
index b2f040a4907..e14db91c5ea 100644
--- a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
+++ b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
@@ -94,6 +94,7 @@ public class RaftClusterInvocationContext implements TestTemplateInvocationConte
                         setBootstrapMetadataVersion(clusterConfig.metadataVersion()).
                         setCombined(isCombined).
                         setNumBrokerNodes(clusterConfig.numBrokers()).
+                        setNumDisksPerBroker(clusterConfig.numDisksPerBroker()).
                         setPerBrokerProperties(clusterConfig.perBrokerOverrideProperties()).
                         setNumControllerNodes(clusterConfig.numControllers()).build();
                 KafkaClusterTestKit.Builder builder = new KafkaClusterTestKit.Builder(nodes);
diff --git a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
index eb88f95a885..7496affa200 100644
--- a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
+++ b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
@@ -373,6 +373,11 @@ public class ZkClusterInvocationContext implements TestTemplateInvocationContext
             return clusterConfig.numBrokers();
         }
 
+        @Override
+        public int logDirCount() {
+            return clusterConfig.numDisksPerBroker();
+        }
+
         @Override
         public Option<File> trustStoreFile() {
             return OptionConverters.toScala(clusterConfig.trustStoreFile());