You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2024/05/03 06:25:06 UTC
(kafka) branch trunk updated: KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)
This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 87390f961f3 KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)
87390f961f3 is described below
commit 87390f961f3bd0c582d0926cb9c4fe00d9298b6f
Author: PoAn Yang <pa...@apache.org>
AuthorDate: Fri May 3 14:24:59 2024 +0800
KAFKA-16572 allow defining number of disks per broker in ClusterTest (#15745)
Reviewers: Luke Chen <sh...@gmail.com>, Chia-Ping Tsai <ch...@gmail.com>
---
core/src/test/java/kafka/test/ClusterConfig.java | 54 +++++++----------
.../test/java/kafka/test/ClusterConfigTest.java | 32 ++++++++--
.../java/kafka/test/ClusterTestExtensionsTest.java | 20 +++++++
.../java/kafka/test/annotation/ClusterTest.java | 1 +
.../kafka/test/annotation/ClusterTestDefaults.java | 1 +
.../kafka/test/junit/ClusterTestExtensions.java | 70 +++++-----------------
.../test/junit/RaftClusterInvocationContext.java | 1 +
.../test/junit/ZkClusterInvocationContext.java | 5 ++
8 files changed, 92 insertions(+), 92 deletions(-)
diff --git a/core/src/test/java/kafka/test/ClusterConfig.java b/core/src/test/java/kafka/test/ClusterConfig.java
index 0bb1ddad481..4a713f35928 100644
--- a/core/src/test/java/kafka/test/ClusterConfig.java
+++ b/core/src/test/java/kafka/test/ClusterConfig.java
@@ -38,6 +38,7 @@ public class ClusterConfig {
private final Type type;
private final int brokers;
private final int controllers;
+ private final int disksPerBroker;
private final String name;
private final boolean autoStart;
@@ -55,14 +56,20 @@ public class ClusterConfig {
private final Map<Integer, Map<String, String>> perBrokerOverrideProperties;
@SuppressWarnings("checkstyle:ParameterNumber")
- private ClusterConfig(Type type, int brokers, int controllers, String name, boolean autoStart,
+ private ClusterConfig(Type type, int brokers, int controllers, int disksPerBroker, String name, boolean autoStart,
SecurityProtocol securityProtocol, String listenerName, File trustStoreFile,
MetadataVersion metadataVersion, Map<String, String> serverProperties, Map<String, String> producerProperties,
Map<String, String> consumerProperties, Map<String, String> adminClientProperties, Map<String, String> saslServerProperties,
Map<String, String> saslClientProperties, Map<Integer, Map<String, String>> perBrokerOverrideProperties) {
+ // do fail fast. the following values are invalid for both zk and kraft modes.
+ if (brokers < 0) throw new IllegalArgumentException("Number of brokers must be greater or equal to zero.");
+ if (controllers < 0) throw new IllegalArgumentException("Number of controller must be greater or equal to zero.");
+ if (disksPerBroker <= 0) throw new IllegalArgumentException("Number of disks must be greater than zero.");
+
this.type = Objects.requireNonNull(type);
this.brokers = brokers;
this.controllers = controllers;
+ this.disksPerBroker = disksPerBroker;
this.name = name;
this.autoStart = autoStart;
this.securityProtocol = Objects.requireNonNull(securityProtocol);
@@ -90,6 +97,10 @@ public class ClusterConfig {
return controllers;
}
+ public int numDisksPerBroker() {
+ return disksPerBroker;
+ }
+
public Optional<String> name() {
return Optional.ofNullable(name);
}
@@ -151,42 +162,12 @@ public class ClusterConfig {
return tags;
}
- @SuppressWarnings({"CyclomaticComplexity"})
- @Override
- public boolean equals(Object object) {
- if (this == object) return true;
- if (object == null || getClass() != object.getClass()) return false;
- ClusterConfig clusterConfig = (ClusterConfig) object;
- return Objects.equals(type, clusterConfig.type)
- && Objects.equals(brokers, clusterConfig.brokers)
- && Objects.equals(controllers, clusterConfig.controllers)
- && Objects.equals(name, clusterConfig.name)
- && Objects.equals(autoStart, clusterConfig.autoStart)
- && Objects.equals(securityProtocol, clusterConfig.securityProtocol)
- && Objects.equals(listenerName, clusterConfig.listenerName)
- && Objects.equals(trustStoreFile, clusterConfig.trustStoreFile)
- && Objects.equals(metadataVersion, clusterConfig.metadataVersion)
- && Objects.equals(serverProperties, clusterConfig.serverProperties)
- && Objects.equals(producerProperties, clusterConfig.producerProperties)
- && Objects.equals(consumerProperties, clusterConfig.consumerProperties)
- && Objects.equals(adminClientProperties, clusterConfig.adminClientProperties)
- && Objects.equals(saslServerProperties, clusterConfig.saslServerProperties)
- && Objects.equals(saslClientProperties, clusterConfig.saslClientProperties)
- && Objects.equals(perBrokerOverrideProperties, clusterConfig.perBrokerOverrideProperties);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(type, brokers, controllers, name, autoStart, securityProtocol, listenerName,
- trustStoreFile, metadataVersion, serverProperties, producerProperties, consumerProperties,
- adminClientProperties, saslServerProperties, saslClientProperties, perBrokerOverrideProperties);
- }
-
public static Builder defaultBuilder() {
return new Builder()
.setType(Type.ZK)
.setBrokers(1)
.setControllers(1)
+ .setDisksPerBroker(1)
.setAutoStart(true)
.setSecurityProtocol(SecurityProtocol.PLAINTEXT)
.setMetadataVersion(MetadataVersion.latestTesting());
@@ -201,6 +182,7 @@ public class ClusterConfig {
.setType(clusterConfig.type)
.setBrokers(clusterConfig.brokers)
.setControllers(clusterConfig.controllers)
+ .setDisksPerBroker(clusterConfig.disksPerBroker)
.setName(clusterConfig.name)
.setAutoStart(clusterConfig.autoStart)
.setSecurityProtocol(clusterConfig.securityProtocol)
@@ -220,6 +202,7 @@ public class ClusterConfig {
private Type type;
private int brokers;
private int controllers;
+ private int disksPerBroker;
private String name;
private boolean autoStart;
private SecurityProtocol securityProtocol;
@@ -251,6 +234,11 @@ public class ClusterConfig {
return this;
}
+ public Builder setDisksPerBroker(int disksPerBroker) {
+ this.disksPerBroker = disksPerBroker;
+ return this;
+ }
+
public Builder setName(String name) {
this.name = name;
return this;
@@ -319,7 +307,7 @@ public class ClusterConfig {
}
public ClusterConfig build() {
- return new ClusterConfig(type, brokers, controllers, name, autoStart, securityProtocol, listenerName,
+ return new ClusterConfig(type, brokers, controllers, disksPerBroker, name, autoStart, securityProtocol, listenerName,
trustStoreFile, metadataVersion, serverProperties, producerProperties, consumerProperties,
adminClientProperties, saslServerProperties, saslClientProperties,
perBrokerOverrideProperties);
diff --git a/core/src/test/java/kafka/test/ClusterConfigTest.java b/core/src/test/java/kafka/test/ClusterConfigTest.java
index 20f9e8f6c33..790ca7d0e5e 100644
--- a/core/src/test/java/kafka/test/ClusterConfigTest.java
+++ b/core/src/test/java/kafka/test/ClusterConfigTest.java
@@ -49,6 +49,7 @@ public class ClusterConfigTest {
.setType(Type.KRAFT)
.setBrokers(3)
.setControllers(2)
+ .setDisksPerBroker(1)
.setName("builder-test")
.setAutoStart(true)
.setSecurityProtocol(SecurityProtocol.PLAINTEXT)
@@ -64,12 +65,35 @@ public class ClusterConfigTest {
.setPerBrokerProperties(Collections.singletonMap(0, Collections.singletonMap("broker_0", "broker_0_value")))
.build();
- ClusterConfig copy = ClusterConfig.builder(clusterConfig).build();
- Assertions.assertEquals(clusterConfig, copy);
- Assertions.assertEquals(clusterConfig.hashCode(), copy.hashCode());
-
Map<String, Object> clusterConfigFields = fields(clusterConfig);
Map<String, Object> copyFields = fields(clusterConfig);
Assertions.assertEquals(clusterConfigFields, copyFields);
}
+
+ @Test
+ public void testBrokerLessThanZero() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+ .setBrokers(-1)
+ .setControllers(1)
+ .setDisksPerBroker(1)
+ .build());
+ }
+
+ @Test
+ public void testControllersLessThanZero() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+ .setBrokers(1)
+ .setControllers(-1)
+ .setDisksPerBroker(1)
+ .build());
+ }
+
+ @Test
+ public void testDisksPerBrokerIsZero() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> ClusterConfig.builder()
+ .setBrokers(1)
+ .setControllers(1)
+ .setDisksPerBroker(0)
+ .build());
+ }
}
diff --git a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
index ada1d1b21b4..0eb94abd660 100644
--- a/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
+++ b/core/src/test/java/kafka/test/ClusterTestExtensionsTest.java
@@ -25,12 +25,15 @@ import kafka.test.annotation.ClusterTestDefaults;
import kafka.test.annotation.ClusterTests;
import kafka.test.annotation.Type;
import kafka.test.junit.ClusterTestExtensions;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.DescribeLogDirsResult;
import org.apache.kafka.server.common.MetadataVersion;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.extension.ExtendWith;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
@ClusterTestDefaults(clusterType = Type.ZK, serverProperties = {
@@ -104,6 +107,23 @@ public class ClusterTestExtensionsTest {
}
}
+ @ClusterTests({
+ @ClusterTest(clusterType = Type.ZK),
+ @ClusterTest(clusterType = Type.ZK, disksPerBroker = 2),
+ @ClusterTest(clusterType = Type.KRAFT),
+ @ClusterTest(clusterType = Type.KRAFT, disksPerBroker = 2),
+ @ClusterTest(clusterType = Type.CO_KRAFT),
+ @ClusterTest(clusterType = Type.CO_KRAFT, disksPerBroker = 2)
+ })
+ public void testClusterTestWithDisksPerBroker() throws ExecutionException, InterruptedException {
+ Admin admin = clusterInstance.createAdminClient();
+
+ DescribeLogDirsResult result = admin.describeLogDirs(clusterInstance.brokerIds());
+ result.allDescriptions().get().forEach((brokerId, logDirDescriptionMap) -> {
+ Assertions.assertEquals(clusterInstance.config().numDisksPerBroker(), logDirDescriptionMap.size());
+ });
+ }
+
@ClusterTest(autoStart = AutoStart.NO)
public void testNoAutoStart() {
Assertions.assertThrows(RuntimeException.class, clusterInstance::anyBrokerSocketServer);
diff --git a/core/src/test/java/kafka/test/annotation/ClusterTest.java b/core/src/test/java/kafka/test/annotation/ClusterTest.java
index 8880966dea8..c3c9364acd0 100644
--- a/core/src/test/java/kafka/test/annotation/ClusterTest.java
+++ b/core/src/test/java/kafka/test/annotation/ClusterTest.java
@@ -36,6 +36,7 @@ public @interface ClusterTest {
Type clusterType() default Type.DEFAULT;
int brokers() default 0;
int controllers() default 0;
+ int disksPerBroker() default 0;
AutoStart autoStart() default AutoStart.DEFAULT;
String name() default "";
diff --git a/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java b/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
index 2f9efd777ef..af45b877652 100644
--- a/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
+++ b/core/src/test/java/kafka/test/annotation/ClusterTestDefaults.java
@@ -38,6 +38,7 @@ public @interface ClusterTestDefaults {
Type clusterType() default Type.ZK;
int brokers() default 1;
int controllers() default 1;
+ int disksPerBroker() default 1;
boolean autoStart() default true;
// Set default server properties for all @ClusterTest(s)
ClusterConfigProperty[] serverProperties() default {};
diff --git a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
index 5a5d78e989c..bdf3b2b9522 100644
--- a/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
+++ b/core/src/test/java/kafka/test/junit/ClusterTestExtensions.java
@@ -19,6 +19,7 @@ package kafka.test.junit;
import kafka.test.ClusterConfig;
import kafka.test.ClusterGenerator;
+import kafka.test.annotation.AutoStart;
import kafka.test.annotation.ClusterTestDefaults;
import kafka.test.annotation.ClusterConfigProperty;
import kafka.test.annotation.ClusterTemplate;
@@ -139,59 +140,7 @@ public class ClusterTestExtensions implements TestTemplateInvocationContextProvi
private void processClusterTest(ExtensionContext context, ClusterTest annot, ClusterTestDefaults defaults,
Consumer<TestTemplateInvocationContext> testInvocations) {
- final Type type;
- if (annot.clusterType() == Type.DEFAULT) {
- type = defaults.clusterType();
- } else {
- type = annot.clusterType();
- }
-
- final int brokers;
- if (annot.brokers() == 0) {
- brokers = defaults.brokers();
- } else {
- brokers = annot.brokers();
- }
-
- final int controllers;
- if (annot.controllers() == 0) {
- controllers = defaults.controllers();
- } else {
- controllers = annot.controllers();
- }
-
- if (brokers <= 0 || controllers <= 0) {
- throw new IllegalArgumentException("Number of brokers/controllers must be greater than zero.");
- }
-
- final boolean autoStart;
- switch (annot.autoStart()) {
- case YES:
- autoStart = true;
- break;
- case NO:
- autoStart = false;
- break;
- case DEFAULT:
- autoStart = defaults.autoStart();
- break;
- default:
- throw new IllegalStateException();
- }
-
- ClusterConfig.Builder configBuilder = ClusterConfig.builder()
- .setType(type)
- .setBrokers(brokers)
- .setControllers(controllers)
- .setAutoStart(autoStart)
- .setSecurityProtocol(annot.securityProtocol())
- .setMetadataVersion(annot.metadataVersion());
- if (!annot.name().isEmpty()) {
- configBuilder.setName(annot.name());
- }
- if (!annot.listener().isEmpty()) {
- configBuilder.setListenerName(annot.listener());
- }
+ Type type = annot.clusterType() == Type.DEFAULT ? defaults.clusterType() : annot.clusterType();
Map<String, String> serverProperties = new HashMap<>();
for (ClusterConfigProperty property : defaults.serverProperties()) {
@@ -200,8 +149,19 @@ public class ClusterTestExtensions implements TestTemplateInvocationContextProvi
for (ClusterConfigProperty property : annot.serverProperties()) {
serverProperties.put(property.key(), property.value());
}
- configBuilder.setServerProperties(serverProperties);
- type.invocationContexts(context.getRequiredTestMethod().getName(), configBuilder.build(), testInvocations);
+ ClusterConfig config = ClusterConfig.builder()
+ .setType(type)
+ .setBrokers(annot.brokers() == 0 ? defaults.brokers() : annot.brokers())
+ .setControllers(annot.controllers() == 0 ? defaults.controllers() : annot.controllers())
+ .setDisksPerBroker(annot.disksPerBroker() == 0 ? defaults.disksPerBroker() : annot.disksPerBroker())
+ .setAutoStart(annot.autoStart() == AutoStart.DEFAULT ? defaults.autoStart() : annot.autoStart() == AutoStart.YES)
+ .setName(annot.name().trim().isEmpty() ? null : annot.name())
+ .setListenerName(annot.listener().trim().isEmpty() ? null : annot.listener())
+ .setServerProperties(serverProperties)
+ .setSecurityProtocol(annot.securityProtocol())
+ .setMetadataVersion(annot.metadataVersion())
+ .build();
+ type.invocationContexts(context.getRequiredTestMethod().getName(), config, testInvocations);
}
private ClusterTestDefaults getClusterTestDefaults(Class<?> testClass) {
diff --git a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
index b2f040a4907..e14db91c5ea 100644
--- a/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
+++ b/core/src/test/java/kafka/test/junit/RaftClusterInvocationContext.java
@@ -94,6 +94,7 @@ public class RaftClusterInvocationContext implements TestTemplateInvocationConte
setBootstrapMetadataVersion(clusterConfig.metadataVersion()).
setCombined(isCombined).
setNumBrokerNodes(clusterConfig.numBrokers()).
+ setNumDisksPerBroker(clusterConfig.numDisksPerBroker()).
setPerBrokerProperties(clusterConfig.perBrokerOverrideProperties()).
setNumControllerNodes(clusterConfig.numControllers()).build();
KafkaClusterTestKit.Builder builder = new KafkaClusterTestKit.Builder(nodes);
diff --git a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
index eb88f95a885..7496affa200 100644
--- a/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
+++ b/core/src/test/java/kafka/test/junit/ZkClusterInvocationContext.java
@@ -373,6 +373,11 @@ public class ZkClusterInvocationContext implements TestTemplateInvocationContext
return clusterConfig.numBrokers();
}
+ @Override
+ public int logDirCount() {
+ return clusterConfig.numDisksPerBroker();
+ }
+
@Override
public Option<File> trustStoreFile() {
return OptionConverters.toScala(clusterConfig.trustStoreFile());