You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2025/01/21 02:25:47 UTC
(kafka) 02/02: KAFKA-14552: Assume a baseline of 3.0 for server protocol versions (#18497)
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch 4.0
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 4ce4ac730127863e8124392f7b2702d81291bd33
Author: Ismael Juma <is...@juma.me.uk>
AuthorDate: Mon Jan 20 13:51:44 2025 -0800
KAFKA-14552: Assume a baseline of 3.0 for server protocol versions (#18497)
Kafka 4.0 will remove support for zk mode and will require conversion to kraft
before upgrading to 4.0. The minimum kraft version is 3.0 (aka 3.0-IV1).
This provides an opportunity to remove exclusively server side protocols versions
that only exist to allow direct upgrades from versions older than 3.0 or that are
used only by zk mode.
Since KRaft became production ready in 3.3, we should consider setting the
baseline to 3.3. But that requires more discussion and it can be done via a
separate change (KAFKA-18601).
Protocol changes:
* Remove RequestHeader v0 (only used by ControlledShutdown v0)
* Remove WriteTxnMarkers v0
* Remove all versions of ControlledShutdown, LeaderAndIsr, StopReplica, UpdateMetadata
In order to remove all versions safely, extend generator to support setting
"versions" to "none". In this case, we no longer generate the `*Data` classes,
but we still reserve the id for the relevant protocol api (so it doesn't get
accidentally used for something else). The protocol documentation is correct
after these changes.
We kept a simplified version of `LeaderAndIsr{Request|Response}` because
it's used by many tests that are still relevant in kraft mode. Once
KAFKA-18486 is done, it may be possible to remove it (I left a comment on
the ticket). Similarly, KAFKA-18487 may make it possible to remove
the introduced `StopReplicaPartitionState` (left a comment on that ticket too).
There are a number of places that were adjusted to include an
`ApiKeys.hasValidVersion` check.
Reviewers: Chia-Ping Tsai <ch...@gmail.com>
---
.../org/apache/kafka/common/protocol/ApiKeys.java | 9 +
.../org/apache/kafka/common/protocol/Protocol.java | 46 +-
.../common/requests/AbstractControlRequest.java | 98 ----
.../kafka/common/requests/AbstractRequest.java | 8 -
.../kafka/common/requests/AbstractResponse.java | 18 +-
.../common/requests/ControlledShutdownRequest.java | 72 ---
.../requests/ControlledShutdownResponse.java | 88 ----
.../kafka/common/requests/LeaderAndIsrRequest.java | 449 +++++++++++-----
.../common/requests/LeaderAndIsrResponse.java | 94 ++--
.../kafka/common/requests/RequestHeader.java | 21 +-
.../kafka/common/requests/StopReplicaRequest.java | 230 --------
.../kafka/common/requests/StopReplicaResponse.java | 88 ----
.../common/requests/UpdateMetadataRequest.java | 255 ---------
.../common/requests/UpdateMetadataResponse.java | 64 ---
.../common/message/AlterPartitionRequest.json | 2 +-
.../common/message/AlterPartitionResponse.json | 2 +-
.../common/message/ControlledShutdownRequest.json | 18 +-
.../common/message/ControlledShutdownResponse.json | 16 +-
.../common/message/LeaderAndIsrRequest.json | 81 +--
.../common/message/LeaderAndIsrResponse.json | 37 +-
.../resources/common/message/RequestHeader.json | 8 +-
.../common/message/StopReplicaRequest.json | 53 +-
.../common/message/StopReplicaResponse.json | 22 +-
.../common/message/UpdateMetadataRequest.json | 87 +--
.../common/message/UpdateMetadataResponse.json | 9 +-
.../common/message/WriteTxnMarkersRequest.json | 4 +-
.../common/message/WriteTxnMarkersResponse.json | 4 +-
.../apache/kafka/clients/NodeApiVersionsTest.java | 2 +
.../kafka/common/message/ApiMessageTypeTest.java | 44 +-
.../apache/kafka/common/message/MessageTest.java | 63 +--
.../apache/kafka/common/protocol/ApiKeysTest.java | 14 +-
.../apache/kafka/common/protocol/ProtocolTest.java | 32 ++
.../requests/ControlledShutdownRequestTest.java | 52 --
.../kafka/common/requests/RequestHeaderTest.java | 26 -
.../kafka/common/requests/RequestResponseTest.java | 260 ---------
core/src/main/scala/kafka/cluster/Partition.scala | 9 +-
core/src/main/scala/kafka/log/LogManager.scala | 45 --
.../main/scala/kafka/network/SocketServer.scala | 4 +-
.../kafka/server/AddPartitionsToTxnManager.scala | 2 +-
.../scala/kafka/server/DelayedElectLeader.scala | 2 +-
core/src/main/scala/kafka/server/KafkaApis.scala | 8 +-
.../main/scala/kafka/server/MetadataCache.scala | 5 +-
.../main/scala/kafka/server/ReplicaManager.scala | 88 +---
.../kafka/server/metadata/KRaftMetadataCache.scala | 17 +-
.../DescribeTopicPartitionsRequestHandlerTest.java | 43 +-
.../kafka/api/AuthorizerIntegrationTest.scala | 11 +-
.../kafka/server/KRaftClusterTest.scala | 12 +-
.../kafka/server/LocalLeaderEndPointTest.scala | 7 +-
.../unit/kafka/cluster/AbstractPartitionTest.scala | 6 +-
.../unit/kafka/cluster/AssignmentStateTest.scala | 4 +-
.../unit/kafka/cluster/PartitionLockTest.scala | 7 +-
.../scala/unit/kafka/cluster/PartitionTest.scala | 119 +++--
.../integration/UncleanLeaderElectionTest.scala | 8 +-
.../test/scala/unit/kafka/log/LogManagerTest.scala | 82 +--
.../scala/unit/kafka/network/ProcessorTest.scala | 31 +-
.../server/AddPartitionsToTxnManagerTest.scala | 13 +-
.../kafka/server/DeleteTopicsRequestTest.scala | 4 +-
.../scala/unit/kafka/server/KafkaApisTest.scala | 2 +-
.../unit/kafka/server/MetadataCacheTest.scala | 16 +-
.../unit/kafka/server/ReplicaManagerTest.scala | 583 +++++----------------
.../scala/unit/kafka/server/RequestQuotaTest.scala | 61 ---
.../server/ShareGroupHeartbeatRequestTest.scala | 2 +-
.../server/epoch/LeaderEpochIntegrationTest.scala | 2 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 11 +-
.../kafka/message/ApiMessageTypeGenerator.java | 60 ++-
.../org/apache/kafka/message/MessageGenerator.java | 65 ++-
.../java/org/apache/kafka/message/MessageSpec.java | 60 ++-
.../apache/kafka/message/MessageGeneratorTest.java | 74 +++
.../apache/kafka/message/StructRegistryTest.java | 18 +
.../org/apache/kafka/message/VersionsTest.java | 1 +
.../jmh/fetcher/ReplicaFetcherThreadBenchmark.java | 4 +-
.../metadata/KRaftMetadataRequestBenchmark.java | 20 +-
.../partition/PartitionMakeFollowerBenchmark.java | 4 +-
.../UpdateFollowerFetchStateBenchmark.java | 4 +-
.../kafka/jmh/server/PartitionCreationBench.java | 4 +-
.../kafka/metadata/PartitionRegistration.java | 8 +-
.../kafka/metadata/PartitionRegistrationTest.java | 6 +-
.../apache/kafka/network/RequestConvertToJson.java | 32 --
.../kafka/network/RequestConvertToJsonTest.java | 57 +-
.../org/apache/kafka/tools/TopicCommandTest.java | 14 +-
80 files changed, 1108 insertions(+), 2903 deletions(-)
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 29b88bee8ee..6afadb8c58e 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -251,6 +251,15 @@ public enum ApiKeys {
return apiVersion >= messageType.lowestDeprecatedVersion() && apiVersion <= messageType.highestDeprecatedVersion();
}
+ /**
+ * Returns `true` if there is at least one valid version, `false` otherwise. When `false` is returned, it typically
+ * means that the protocol api is no longer supported, but the api key remains assigned to the removed api so we
+ * do not accidentally reuse it for a different api.
+ */
+ public boolean hasValidVersion() {
+ return oldestVersion() <= latestVersion();
+ }
+
public Optional<ApiVersionsResponseData.ApiVersion> toApiVersion(boolean enableUnstableLastVersion) {
short oldestVersion = oldestVersion();
short latestVersion = latestVersion(enableUnstableLastVersion);
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
index 614c8cd22e4..1459a901030 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
@@ -153,14 +153,14 @@ public class Protocol {
final StringBuilder b = new StringBuilder();
b.append("<h5>Headers:</h5>\n");
- for (int i = 0; i < RequestHeaderData.SCHEMAS.length; i++) {
+ for (int i = RequestHeaderData.LOWEST_SUPPORTED_VERSION; i <= RequestHeaderData.HIGHEST_SUPPORTED_VERSION; i++) {
b.append("<pre>");
b.append("Request Header v").append(i).append(" => ");
schemaToBnfHtml(RequestHeaderData.SCHEMAS[i], b, 2);
b.append("</pre>\n");
schemaToFieldTableHtml(RequestHeaderData.SCHEMAS[i], b);
}
- for (int i = 0; i < ResponseHeaderData.SCHEMAS.length; i++) {
+ for (int i = ResponseHeaderData.LOWEST_SUPPORTED_VERSION; i <= ResponseHeaderData.HIGHEST_SUPPORTED_VERSION; i++) {
b.append("<pre>");
b.append("Response Header v").append(i).append(" => ");
schemaToBnfHtml(ResponseHeaderData.SCHEMAS[i], b, 2);
@@ -178,30 +178,30 @@ public class Protocol {
// Requests
b.append("<b>Requests:</b><br>\n");
Schema[] requests = key.messageType.requestSchemas();
- for (int i = 0; i < requests.length; i++) {
- Schema schema = requests[i];
+ for (short version = key.oldestVersion(); version <= key.latestVersion(); version++) {
+ Schema schema = requests[version];
+ if (schema == null)
+ throw new IllegalStateException("Unexpected null schema for " + key + " with version " + version);
// Schema
- if (schema != null) {
- b.append("<div>");
- // Version header
- b.append("<pre>");
- b.append(key.name);
- b.append(" Request (Version: ");
- b.append(i);
- b.append(") => ");
- schemaToBnfHtml(requests[i], b, 2);
- b.append("</pre>");
-
- if (!key.isVersionEnabled((short) i, false)) {
- b.append("<p>This version of the request is unstable.</p>");
- }
+ b.append("<div>");
+ // Version header
+ b.append("<pre>");
+ b.append(key.name);
+ b.append(" Request (Version: ");
+ b.append(version);
+ b.append(") => ");
+ schemaToBnfHtml(schema, b, 2);
+ b.append("</pre>");
+
+ if (!key.isVersionEnabled(version, false)) {
+ b.append("<p>This version of the request is unstable.</p>");
+ }
- b.append("<p><b>Request header version:</b> ");
- b.append(key.requestHeaderVersion((short) i));
- b.append("</p>\n");
+ b.append("<p><b>Request header version:</b> ");
+ b.append(key.requestHeaderVersion(version));
+ b.append("</p>\n");
- schemaToFieldTableHtml(requests[i], b);
- }
+ schemaToFieldTableHtml(schema, b);
b.append("</div>\n");
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
deleted file mode 100644
index e2fc69e3bff..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractControlRequest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.protocol.ApiKeys;
-
-// Abstract class for all control requests including UpdateMetadataRequest, LeaderAndIsrRequest and StopReplicaRequest
-public abstract class AbstractControlRequest extends AbstractRequest {
-
- /**
- * Indicates if a controller request is incremental, full, or unknown.
- * Used by LeaderAndIsrRequest.Type and UpdateMetadataRequest.Type fields.
- */
- public enum Type {
- UNKNOWN(0),
- INCREMENTAL(1),
- FULL(2);
-
- private final byte type;
- Type(int type) {
- this.type = (byte) type;
- }
-
- public byte toByte() {
- return type;
- }
-
- public static Type fromByte(byte type) {
- for (Type t : Type.values()) {
- if (t.type == type) {
- return t;
- }
- }
- return UNKNOWN;
- }
- }
-
- public static final long UNKNOWN_BROKER_EPOCH = -1L;
-
- public abstract static class Builder<T extends AbstractRequest> extends AbstractRequest.Builder<T> {
- protected final int controllerId;
- protected final int controllerEpoch;
- protected final long brokerEpoch;
- protected final boolean kraftController;
-
- protected Builder(ApiKeys api, short version, int controllerId, int controllerEpoch, long brokerEpoch) {
- this(api, version, controllerId, controllerEpoch, brokerEpoch, false);
- }
-
- protected Builder(ApiKeys api, short version, int controllerId, int controllerEpoch,
- long brokerEpoch, boolean kraftController) {
- super(api, version);
- this.controllerId = controllerId;
- this.controllerEpoch = controllerEpoch;
- this.brokerEpoch = brokerEpoch;
- this.kraftController = kraftController;
- }
-
- public int controllerId() {
- return controllerId;
- }
-
- public int controllerEpoch() {
- return controllerEpoch;
- }
-
- public long brokerEpoch() {
- return brokerEpoch;
- }
- }
-
- protected AbstractControlRequest(ApiKeys api, short version) {
- super(api, version);
- }
-
- public abstract int controllerId();
-
- public abstract boolean isKRaftController();
-
- public abstract int controllerEpoch();
-
- public abstract long brokerEpoch();
-
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index 87a7a826869..38da78efb1b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -196,14 +196,6 @@ public abstract class AbstractRequest implements AbstractRequestResponse {
return LeaveGroupRequest.parse(buffer, apiVersion);
case SYNC_GROUP:
return SyncGroupRequest.parse(buffer, apiVersion);
- case STOP_REPLICA:
- return StopReplicaRequest.parse(buffer, apiVersion);
- case CONTROLLED_SHUTDOWN:
- return ControlledShutdownRequest.parse(buffer, apiVersion);
- case UPDATE_METADATA:
- return UpdateMetadataRequest.parse(buffer, apiVersion);
- case LEADER_AND_ISR:
- return LeaderAndIsrRequest.parse(buffer, apiVersion);
case DESCRIBE_GROUPS:
return DescribeGroupsRequest.parse(buffer, apiVersion);
case LIST_GROUPS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
index 83f29471ba3..cc29f4a0cd0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -62,29 +62,29 @@ public abstract class AbstractResponse implements AbstractRequestResponse {
*/
public abstract Map<Errors, Integer> errorCounts();
- protected Map<Errors, Integer> errorCounts(Errors error) {
+ protected static Map<Errors, Integer> errorCounts(Errors error) {
return Collections.singletonMap(error, 1);
}
- protected Map<Errors, Integer> errorCounts(Stream<Errors> errors) {
+ protected static Map<Errors, Integer> errorCounts(Stream<Errors> errors) {
return errors.collect(Collectors.groupingBy(e -> e, Collectors.summingInt(e -> 1)));
}
- protected Map<Errors, Integer> errorCounts(Collection<Errors> errors) {
+ protected static Map<Errors, Integer> errorCounts(Collection<Errors> errors) {
Map<Errors, Integer> errorCounts = new HashMap<>();
for (Errors error : errors)
updateErrorCounts(errorCounts, error);
return errorCounts;
}
- protected Map<Errors, Integer> apiErrorCounts(Map<?, ApiError> errors) {
+ protected static Map<Errors, Integer> apiErrorCounts(Map<?, ApiError> errors) {
Map<Errors, Integer> errorCounts = new HashMap<>();
for (ApiError apiError : errors.values())
updateErrorCounts(errorCounts, apiError.error());
return errorCounts;
}
- protected void updateErrorCounts(Map<Errors, Integer> errorCounts, Errors error) {
+ protected static void updateErrorCounts(Map<Errors, Integer> errorCounts, Errors error) {
Integer count = errorCounts.getOrDefault(error, 0);
errorCounts.put(error, count + 1);
}
@@ -133,14 +133,6 @@ public abstract class AbstractResponse implements AbstractRequestResponse {
return LeaveGroupResponse.parse(responseBuffer, version);
case SYNC_GROUP:
return SyncGroupResponse.parse(responseBuffer, version);
- case STOP_REPLICA:
- return StopReplicaResponse.parse(responseBuffer, version);
- case CONTROLLED_SHUTDOWN:
- return ControlledShutdownResponse.parse(responseBuffer, version);
- case UPDATE_METADATA:
- return UpdateMetadataResponse.parse(responseBuffer, version);
- case LEADER_AND_ISR:
- return LeaderAndIsrResponse.parse(responseBuffer, version);
case DESCRIBE_GROUPS:
return DescribeGroupsResponse.parse(responseBuffer, version);
case LIST_GROUPS:
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
deleted file mode 100644
index 088c351302f..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.message.ControlledShutdownRequestData;
-import org.apache.kafka.common.message.ControlledShutdownResponseData;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-
-import java.nio.ByteBuffer;
-
-public class ControlledShutdownRequest extends AbstractRequest {
-
- public static class Builder extends AbstractRequest.Builder<ControlledShutdownRequest> {
-
- private final ControlledShutdownRequestData data;
-
- public Builder(ControlledShutdownRequestData data, short desiredVersion) {
- super(ApiKeys.CONTROLLED_SHUTDOWN, desiredVersion);
- this.data = data;
- }
-
- @Override
- public ControlledShutdownRequest build(short version) {
- return new ControlledShutdownRequest(data, version);
- }
-
- @Override
- public String toString() {
- return data.toString();
- }
- }
-
- private final ControlledShutdownRequestData data;
-
- private ControlledShutdownRequest(ControlledShutdownRequestData data, short version) {
- super(ApiKeys.CONTROLLED_SHUTDOWN, version);
- this.data = data;
- }
-
- @Override
- public ControlledShutdownResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- ControlledShutdownResponseData data = new ControlledShutdownResponseData()
- .setErrorCode(Errors.forException(e).code());
- return new ControlledShutdownResponse(data);
- }
-
- public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) {
- return new ControlledShutdownRequest(new ControlledShutdownRequestData(new ByteBufferAccessor(buffer), version),
- version);
- }
-
- @Override
- public ControlledShutdownRequestData data() {
- return data;
- }
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
deleted file mode 100644
index d3b606eeac6..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.message.ControlledShutdownResponseData;
-import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.Set;
-
-
-public class ControlledShutdownResponse extends AbstractResponse {
-
- /**
- * Possible error codes:
- *
- * UNKNOWN(-1) (this is because IllegalStateException may be thrown in `KafkaController.shutdownBroker`, it would be good to improve this)
- * BROKER_NOT_AVAILABLE(8)
- * STALE_CONTROLLER_EPOCH(11)
- */
- private final ControlledShutdownResponseData data;
-
- public ControlledShutdownResponse(ControlledShutdownResponseData data) {
- super(ApiKeys.CONTROLLED_SHUTDOWN);
- this.data = data;
- }
-
- public Errors error() {
- return Errors.forCode(data.errorCode());
- }
-
- @Override
- public Map<Errors, Integer> errorCounts() {
- return errorCounts(error());
- }
-
- @Override
- public int throttleTimeMs() {
- return DEFAULT_THROTTLE_TIME;
- }
-
- @Override
- public void maybeSetThrottleTimeMs(int throttleTimeMs) {
- // Not supported by the response schema
- }
-
- public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) {
- return new ControlledShutdownResponse(new ControlledShutdownResponseData(new ByteBufferAccessor(buffer), version));
- }
-
- @Override
- public ControlledShutdownResponseData data() {
- return data;
- }
-
- public static ControlledShutdownResponse prepareResponse(Errors error, Set<TopicPartition> tps) {
- ControlledShutdownResponseData data = new ControlledShutdownResponseData();
- data.setErrorCode(error.code());
- ControlledShutdownResponseData.RemainingPartitionCollection pSet = new ControlledShutdownResponseData.RemainingPartitionCollection();
- tps.forEach(tp ->
- pSet.add(new RemainingPartition()
- .setTopicName(tp.topic())
- .setPartitionIndex(tp.partition()))
- );
- data.setRemainingPartitions(pSet);
- return new ControlledShutdownResponse(data);
- }
-
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index c812e29f664..90c0add47a9 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -18,96 +18,69 @@ package org.apache.kafka.common.requests;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrTopicState;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.MessageUtil;
import org.apache.kafka.common.utils.FlattenedIterator;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
-public final class LeaderAndIsrRequest extends AbstractControlRequest {
+public final class LeaderAndIsrRequest {
- public static class Builder extends AbstractControlRequest.Builder<LeaderAndIsrRequest> {
+ /**
+ * Indicates if a controller request is incremental, full, or unknown.
+ */
+ public enum Type {
+ UNKNOWN(0),
+ INCREMENTAL(1),
+ FULL(2);
- private final List<LeaderAndIsrPartitionState> partitionStates;
+ private final byte type;
+ Type(int type) {
+ this.type = (byte) type;
+ }
+
+ public byte toByte() {
+ return type;
+ }
+ }
+
+ public static class Builder {
+ protected final int controllerId;
+ protected final int controllerEpoch;
+ protected final long brokerEpoch;
+ private final List<PartitionState> partitionStates;
private final Map<String, Uuid> topicIds;
private final Collection<Node> liveLeaders;
private final Type updateType;
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- List<LeaderAndIsrPartitionState> partitionStates, Map<String, Uuid> topicIds,
+ public Builder(int controllerId, int controllerEpoch, long brokerEpoch,
+ List<PartitionState> partitionStates, Map<String, Uuid> topicIds,
Collection<Node> liveLeaders) {
- this(version, controllerId, controllerEpoch, brokerEpoch, partitionStates, topicIds,
- liveLeaders, false, Type.UNKNOWN);
+ this(controllerId, controllerEpoch, brokerEpoch, partitionStates, topicIds, liveLeaders, Type.UNKNOWN);
}
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- List<LeaderAndIsrPartitionState> partitionStates, Map<String, Uuid> topicIds,
- Collection<Node> liveLeaders, boolean kraftController, Type updateType) {
- super(ApiKeys.LEADER_AND_ISR, version, controllerId, controllerEpoch, brokerEpoch, kraftController);
+ public Builder(int controllerId, int controllerEpoch, long brokerEpoch,
+ List<PartitionState> partitionStates, Map<String, Uuid> topicIds,
+ Collection<Node> liveLeaders, Type updateType) {
+ this.controllerId = controllerId;
+ this.controllerEpoch = controllerEpoch;
+ this.brokerEpoch = brokerEpoch;
this.partitionStates = partitionStates;
this.topicIds = topicIds;
this.liveLeaders = liveLeaders;
this.updateType = updateType;
}
- @Override
- public LeaderAndIsrRequest build(short version) {
- List<LeaderAndIsrLiveLeader> leaders = liveLeaders.stream().map(n -> new LeaderAndIsrLiveLeader()
- .setBrokerId(n.id())
- .setHostName(n.host())
- .setPort(n.port())
- ).collect(Collectors.toList());
-
- LeaderAndIsrRequestData data = new LeaderAndIsrRequestData()
- .setControllerId(controllerId)
- .setControllerEpoch(controllerEpoch)
- .setBrokerEpoch(brokerEpoch)
- .setLiveLeaders(leaders);
-
- if (version >= 7) {
- data.setIsKRaftController(kraftController);
- }
-
- if (version >= 5) {
- data.setType(updateType.toByte());
- }
-
- if (version >= 2) {
- Map<String, LeaderAndIsrTopicState> topicStatesMap = groupByTopic(partitionStates, topicIds);
- data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
- } else {
- data.setUngroupedPartitionStates(partitionStates);
- }
-
- return new LeaderAndIsrRequest(data, version);
- }
-
- private static Map<String, LeaderAndIsrTopicState> groupByTopic(List<LeaderAndIsrPartitionState> partitionStates, Map<String, Uuid> topicIds) {
- Map<String, LeaderAndIsrTopicState> topicStates = new HashMap<>();
- // We don't null out the topic name in LeaderAndIsrRequestPartition since it's ignored by
- // the generated code if version >= 2
- for (LeaderAndIsrPartitionState partition : partitionStates) {
- LeaderAndIsrTopicState topicState = topicStates.computeIfAbsent(partition.topicName(), t -> new LeaderAndIsrTopicState()
- .setTopicName(partition.topicName())
- .setTopicId(topicIds.getOrDefault(partition.topicName(), Uuid.ZERO_UUID)));
- topicState.partitionStates().add(partition);
- }
- return topicStates;
+ public LeaderAndIsrRequest build() {
+ return new LeaderAndIsrRequest(this);
}
@Override
@@ -124,105 +97,307 @@ public final class LeaderAndIsrRequest extends AbstractControlRequest {
}
}
- private final LeaderAndIsrRequestData data;
-
- public LeaderAndIsrRequest(LeaderAndIsrRequestData data, short version) {
- super(ApiKeys.LEADER_AND_ISR, version);
- this.data = data;
- // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
- normalize();
+ private final int controllerId;
+ private final int controllerEpoch;
+ private final long brokerEpoch;
+ private final List<Node> liveLeaders;
+ private final List<TopicState> topicStates;
+ private final Type requestType;
+
+ public LeaderAndIsrRequest(Builder builder) {
+ this.controllerId = builder.controllerId;
+ this.controllerEpoch = builder.controllerEpoch;
+ this.brokerEpoch = builder.brokerEpoch;
+ this.requestType = builder.updateType;
+ this.liveLeaders = new ArrayList<>(builder.liveLeaders);
+ this.topicStates = new ArrayList<>(groupByTopic(builder.partitionStates, builder.topicIds).values());
}
- private void normalize() {
- if (version() >= 2) {
- for (LeaderAndIsrTopicState topicState : data.topicStates()) {
- for (LeaderAndIsrPartitionState partitionState : topicState.partitionStates()) {
- // Set the topic name so that we can always present the ungrouped view to callers
- partitionState.setTopicName(topicState.topicName());
- }
- }
- }
- }
-
- @Override
- public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData();
- Errors error = Errors.forException(e);
- responseData.setErrorCode(error.code());
-
- if (version() < 5) {
- List<LeaderAndIsrPartitionError> partitions = new ArrayList<>();
- for (LeaderAndIsrPartitionState partition : partitionStates()) {
- partitions.add(new LeaderAndIsrPartitionError()
- .setTopicName(partition.topicName())
- .setPartitionIndex(partition.partitionIndex())
- .setErrorCode(error.code()));
- }
- responseData.setPartitionErrors(partitions);
- } else {
- for (LeaderAndIsrTopicState topicState : data.topicStates()) {
- List<LeaderAndIsrPartitionError> partitions = new ArrayList<>(
- topicState.partitionStates().size());
- for (LeaderAndIsrPartitionState partition : topicState.partitionStates()) {
- partitions.add(new LeaderAndIsrPartitionError()
- .setPartitionIndex(partition.partitionIndex())
- .setErrorCode(error.code()));
- }
- responseData.topics().add(new LeaderAndIsrTopicError()
- .setTopicId(topicState.topicId())
- .setPartitionErrors(partitions));
- }
+ private static Map<String, TopicState> groupByTopic(List<PartitionState> partitionStates, Map<String, Uuid> topicIds) {
+ Map<String, TopicState> topicStates = new HashMap<>();
+ for (PartitionState partition : partitionStates) {
+ TopicState topicState = topicStates.computeIfAbsent(partition.topicName(), t -> {
+ var topic = new TopicState();
+ topic.topicName = partition.topicName();
+ topic.topicId = topicIds.getOrDefault(partition.topicName(), Uuid.ZERO_UUID);
+ return topic;
+ });
+ topicState.partitionStates().add(partition);
}
-
- return new LeaderAndIsrResponse(responseData, version());
+ return topicStates;
}
- @Override
public int controllerId() {
- return data.controllerId();
+ return controllerId;
}
- @Override
- public boolean isKRaftController() {
- return data.isKRaftController();
- }
-
- @Override
public int controllerEpoch() {
- return data.controllerEpoch();
+ return controllerEpoch;
}
- @Override
public long brokerEpoch() {
- return data.brokerEpoch();
+ return brokerEpoch;
}
- public Iterable<LeaderAndIsrPartitionState> partitionStates() {
- if (version() >= 2)
- return () -> new FlattenedIterator<>(data.topicStates().iterator(),
+ public Iterable<PartitionState> partitionStates() {
+ return () -> new FlattenedIterator<>(topicStates.iterator(),
topicState -> topicState.partitionStates().iterator());
- return data.ungroupedPartitionStates();
}
public Map<String, Uuid> topicIds() {
- return data.topicStates().stream()
- .collect(Collectors.toMap(LeaderAndIsrTopicState::topicName, LeaderAndIsrTopicState::topicId));
+ return topicStates.stream()
+ .collect(Collectors.toMap(TopicState::topicName, TopicState::topicId));
}
- public List<LeaderAndIsrLiveLeader> liveLeaders() {
- return Collections.unmodifiableList(data.liveLeaders());
+ public List<Node> liveLeaders() {
+ return Collections.unmodifiableList(liveLeaders);
}
public Type requestType() {
- return Type.fromByte(data.type());
+ return requestType;
+ }
+
+ public LeaderAndIsrResponse getErrorResponse(Exception e) {
+ LinkedHashMap<Uuid, List<LeaderAndIsrResponse.PartitionError>> errorsMap = new LinkedHashMap<>();
+ Errors error = Errors.forException(e);
+
+ for (TopicState topicState : topicStates) {
+ List<LeaderAndIsrResponse.PartitionError> partitions = new ArrayList<>(topicState.partitionStates().size());
+ for (PartitionState partition : topicState.partitionStates()) {
+ partitions.add(new LeaderAndIsrResponse.PartitionError(partition.partitionIndex, error.code()));
+ }
+ errorsMap.put(topicState.topicId, partitions);
+ }
+
+ return new LeaderAndIsrResponse(error, errorsMap);
+
}
- @Override
- public LeaderAndIsrRequestData data() {
- return data;
+ public static class TopicState {
+ String topicName;
+ Uuid topicId;
+ List<PartitionState> partitionStates;
+
+ public TopicState() {
+ this.topicName = "";
+ this.topicId = Uuid.ZERO_UUID;
+ this.partitionStates = new ArrayList<>(0);
+ }
+
+ public String topicName() {
+ return this.topicName;
+ }
+
+ public Uuid topicId() {
+ return this.topicId;
+ }
+
+ public List<PartitionState> partitionStates() {
+ return this.partitionStates;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass())
+ return false;
+ TopicState that = (TopicState) o;
+ return Objects.equals(topicName, that.topicName) &&
+ Objects.equals(topicId, that.topicId) &&
+ Objects.equals(partitionStates, that.partitionStates);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(topicName, topicId, partitionStates);
+ }
+
+ @Override
+ public String toString() {
+ return "LeaderAndIsrTopicState("
+ + "topicName='" + topicName + "'"
+ + ", topicId=" + topicId
+ + ", partitionStates=" + MessageUtil.deepToString(partitionStates.iterator())
+ + ")";
+ }
}
- public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) {
- return new LeaderAndIsrRequest(new LeaderAndIsrRequestData(new ByteBufferAccessor(buffer), version), version);
+ public static class PartitionState {
+ String topicName;
+ int partitionIndex;
+ int controllerEpoch;
+ int leader;
+ int leaderEpoch;
+ List<Integer> isr;
+ int partitionEpoch;
+ List<Integer> replicas;
+ List<Integer> addingReplicas;
+ List<Integer> removingReplicas;
+ boolean isNew;
+ byte leaderRecoveryState;
+
+ public PartitionState() {
+ this.topicName = "";
+ this.partitionIndex = 0;
+ this.controllerEpoch = 0;
+ this.leader = 0;
+ this.leaderEpoch = 0;
+ this.isr = new ArrayList<>(0);
+ this.partitionEpoch = 0;
+ this.replicas = new ArrayList<>(0);
+ this.addingReplicas = new ArrayList<>(0);
+ this.removingReplicas = new ArrayList<>(0);
+ this.isNew = false;
+ this.leaderRecoveryState = (byte) 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) return false;
+ PartitionState that = (PartitionState) o;
+ return partitionIndex == that.partitionIndex &&
+ controllerEpoch == that.controllerEpoch &&
+ leader == that.leader &&
+ leaderEpoch == that.leaderEpoch &&
+ partitionEpoch == that.partitionEpoch &&
+ isNew == that.isNew &&
+ leaderRecoveryState == that.leaderRecoveryState &&
+ Objects.equals(topicName, that.topicName) &&
+ Objects.equals(isr, that.isr) &&
+ Objects.equals(replicas, that.replicas) &&
+ Objects.equals(addingReplicas, that.addingReplicas) &&
+ Objects.equals(removingReplicas, that.removingReplicas);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(topicName, partitionIndex, controllerEpoch, leader, leaderEpoch, isr, partitionEpoch,
+ replicas, addingReplicas, removingReplicas, isNew, leaderRecoveryState);
+ }
+
+ @Override
+ public String toString() {
+ return "LeaderAndIsrPartitionState("
+ + "topicName='" + topicName + "'"
+ + ", partitionIndex=" + partitionIndex
+ + ", controllerEpoch=" + controllerEpoch
+ + ", leader=" + leader
+ + ", leaderEpoch=" + leaderEpoch
+ + ", isr=" + MessageUtil.deepToString(isr.iterator())
+ + ", partitionEpoch=" + partitionEpoch
+ + ", replicas=" + MessageUtil.deepToString(replicas.iterator())
+ + ", addingReplicas=" + MessageUtil.deepToString(addingReplicas.iterator())
+ + ", removingReplicas=" + MessageUtil.deepToString(removingReplicas.iterator())
+ + ", isNew=" + (isNew ? "true" : "false")
+ + ", leaderRecoveryState=" + leaderRecoveryState
+ + ")";
+ }
+
+ public String topicName() {
+ return this.topicName;
+ }
+
+ public int partitionIndex() {
+ return this.partitionIndex;
+ }
+
+ public int controllerEpoch() {
+ return this.controllerEpoch;
+ }
+
+ public int leader() {
+ return this.leader;
+ }
+
+ public int leaderEpoch() {
+ return this.leaderEpoch;
+ }
+
+ public List<Integer> isr() {
+ return this.isr;
+ }
+
+ public int partitionEpoch() {
+ return this.partitionEpoch;
+ }
+
+ public List<Integer> replicas() {
+ return this.replicas;
+ }
+
+ public List<Integer> addingReplicas() {
+ return this.addingReplicas;
+ }
+
+ public List<Integer> removingReplicas() {
+ return this.removingReplicas;
+ }
+
+ public boolean isNew() {
+ return this.isNew;
+ }
+
+ public byte leaderRecoveryState() {
+ return this.leaderRecoveryState;
+ }
+
+ public PartitionState setTopicName(String v) {
+ this.topicName = v;
+ return this;
+ }
+
+ public PartitionState setPartitionIndex(int v) {
+ this.partitionIndex = v;
+ return this;
+ }
+
+ public PartitionState setControllerEpoch(int v) {
+ this.controllerEpoch = v;
+ return this;
+ }
+
+ public PartitionState setLeader(int v) {
+ this.leader = v;
+ return this;
+ }
+
+ public PartitionState setLeaderEpoch(int v) {
+ this.leaderEpoch = v;
+ return this;
+ }
+
+ public PartitionState setIsr(List<Integer> v) {
+ this.isr = v;
+ return this;
+ }
+
+ public PartitionState setPartitionEpoch(int v) {
+ this.partitionEpoch = v;
+ return this;
+ }
+
+ public PartitionState setReplicas(List<Integer> v) {
+ this.replicas = v;
+ return this;
+ }
+
+ public PartitionState setAddingReplicas(List<Integer> v) {
+ this.addingReplicas = v;
+ return this;
+ }
+
+ public PartitionState setRemovingReplicas(List<Integer> v) {
+ this.removingReplicas = v;
+ return this;
+ }
+
+ public PartitionState setIsNew(boolean v) {
+ this.isNew = v;
+ return this;
+ }
+
+ public PartitionState setLeaderRecoveryState(byte v) {
+ this.leaderRecoveryState = v;
+ return this;
+ }
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
index 0d40581d68e..bcdcf35b5d3 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
@@ -18,19 +18,15 @@ package org.apache.kafka.common.requests;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
import org.apache.kafka.common.protocol.Errors;
-import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
-public class LeaderAndIsrResponse extends AbstractResponse {
+public class LeaderAndIsrResponse {
/**
* Possible error code:
@@ -38,84 +34,62 @@ public class LeaderAndIsrResponse extends AbstractResponse {
* STALE_CONTROLLER_EPOCH (11)
* STALE_BROKER_EPOCH (77)
*/
- private final LeaderAndIsrResponseData data;
- private final short version;
+ private final Errors error;
+ private final LinkedHashMap<Uuid, List<PartitionError>> topicErrors;
- public LeaderAndIsrResponse(LeaderAndIsrResponseData data, short version) {
- super(ApiKeys.LEADER_AND_ISR);
- this.data = data;
- this.version = version;
+ public LeaderAndIsrResponse(Errors error, LinkedHashMap<Uuid, List<PartitionError>> topicErrors) {
+ this.error = error;
+ this.topicErrors = topicErrors;
}
- public LeaderAndIsrTopicErrorCollection topics() {
- return this.data.topics();
+ public LinkedHashMap<Uuid, List<PartitionError>> topics() {
+ return topicErrors;
}
public Errors error() {
- return Errors.forCode(data.errorCode());
+ return error;
}
- @Override
public Map<Errors, Integer> errorCounts() {
Errors error = error();
if (error != Errors.NONE) {
// Minor optimization since the top-level error applies to all partitions
- if (version < 5)
- return Collections.singletonMap(error, data.partitionErrors().size() + 1);
- return Collections.singletonMap(error,
- data.topics().stream().mapToInt(t -> t.partitionErrors().size()).sum() + 1);
+ return Collections.singletonMap(error, topics().values().stream().mapToInt(partitionErrors ->
+ partitionErrors.size()).sum() + 1);
}
- Map<Errors, Integer> errors;
- if (version < 5)
- errors = errorCounts(data.partitionErrors().stream().map(l -> Errors.forCode(l.errorCode())));
- else
- errors = errorCounts(data.topics().stream().flatMap(t -> t.partitionErrors().stream()).map(l ->
- Errors.forCode(l.errorCode())));
- updateErrorCounts(errors, Errors.NONE);
+ Map<Errors, Integer> errors = AbstractResponse.errorCounts(topics().values().stream().flatMap(partitionErrors ->
+ partitionErrors.stream()).map(p -> Errors.forCode(p.errorCode)));
+ AbstractResponse.updateErrorCounts(errors, Errors.NONE);
return errors;
}
public Map<TopicPartition, Errors> partitionErrors(Map<Uuid, String> topicNames) {
Map<TopicPartition, Errors> errors = new HashMap<>();
- if (version < 5) {
- data.partitionErrors().forEach(partition ->
- errors.put(new TopicPartition(partition.topicName(), partition.partitionIndex()),
- Errors.forCode(partition.errorCode())));
- } else {
- for (LeaderAndIsrTopicError topic : data.topics()) {
- String topicName = topicNames.get(topic.topicId());
- if (topicName != null) {
- topic.partitionErrors().forEach(partition ->
- errors.put(new TopicPartition(topicName, partition.partitionIndex()),
- Errors.forCode(partition.errorCode())));
- }
+ topics().forEach((topicId, partitionErrors) -> {
+ String topicName = topicNames.get(topicId);
+ if (topicName != null) {
+ partitionErrors.forEach(partition ->
+ errors.put(new TopicPartition(topicName, partition.partitionIndex), Errors.forCode(partition.errorCode)));
}
- }
+ });
return errors;
}
@Override
- public int throttleTimeMs() {
- return DEFAULT_THROTTLE_TIME;
- }
-
- @Override
- public void maybeSetThrottleTimeMs(int throttleTimeMs) {
- // Not supported by the response schema
- }
-
- public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) {
- return new LeaderAndIsrResponse(new LeaderAndIsrResponseData(new ByteBufferAccessor(buffer), version), version);
+ public String toString() {
+ return "LeaderAndIsrResponse{" +
+ "error=" + error +
+ ", topicErrors=" + topicErrors +
+ '}';
}
- @Override
- public LeaderAndIsrResponseData data() {
- return data;
- }
+ public static class PartitionError {
+ public final int partitionIndex;
+ public final short errorCode;
- @Override
- public String toString() {
- return data.toString();
+ public PartitionError(int partitionIndex, short errorCode) {
+ this.partitionIndex = partitionIndex;
+ this.errorCode = errorCode;
+ }
}
-
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
index 45063b816bc..a628811cde8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java
@@ -121,14 +121,21 @@ public class RequestHeader implements AbstractRequestResponse {
}
public static RequestHeader parse(ByteBuffer buffer) {
- short apiKey = -1;
+ short apiKeyId = -1;
try {
// We derive the header version from the request api version, so we read that first.
// The request api version is part of `RequestHeaderData`, so we reset the buffer position after the read.
int bufferStartPositionForHeader = buffer.position();
- apiKey = buffer.getShort();
+ apiKeyId = buffer.getShort();
short apiVersion = buffer.getShort();
- short headerVersion = ApiKeys.forId(apiKey).requestHeaderVersion(apiVersion);
+ ApiKeys apiKey = ApiKeys.forId(apiKeyId);
+
+ // `apiKey.requestHeaderVersion` will fail if there are no valid versions - we do this check first in order to
+ // provide a more helpful message
+ if (!apiKey.hasValidVersion())
+ throw new InvalidRequestException("Unsupported api with key " + apiKeyId + " (" + apiKey.name + ") and version " + apiVersion);
+
+ short headerVersion = apiKey.requestHeaderVersion(apiVersion);
buffer.position(bufferStartPositionForHeader);
final RequestHeaderData headerData = new RequestHeaderData(new ByteBufferAccessor(buffer), headerVersion);
// Due to a quirk in the protocol, client ID is marked as nullable.
@@ -144,10 +151,12 @@ public class RequestHeader implements AbstractRequestResponse {
header.size = Math.max(buffer.position() - bufferStartPositionForHeader, 0);
return header;
} catch (UnsupportedVersionException e) {
- throw new InvalidRequestException("Unknown API key " + apiKey, e);
+ throw new InvalidRequestException("Unknown API key " + apiKeyId, e);
+ } catch (InvalidRequestException e) {
+ throw e;
} catch (Throwable ex) {
- throw new InvalidRequestException("Error parsing request header. Our best guess of the apiKey is: " +
- apiKey, ex);
+ throw new InvalidRequestException("Error parsing request header. Our best guess of the apiKeyId is: " +
+ apiKeyId, ex);
}
}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
deleted file mode 100644
index 92414712658..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.message.StopReplicaRequestData;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionState;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionV0;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopicState;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopicV1;
-import org.apache.kafka.common.message.StopReplicaResponseData;
-import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.utils.MappedIterator;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class StopReplicaRequest extends AbstractControlRequest {
-
- public static class Builder extends AbstractControlRequest.Builder<StopReplicaRequest> {
- private final boolean deletePartitions;
- private final List<StopReplicaTopicState> topicStates;
-
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- boolean deletePartitions, List<StopReplicaTopicState> topicStates) {
- this(version, controllerId, controllerEpoch, brokerEpoch, deletePartitions,
- topicStates, false);
- }
-
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- boolean deletePartitions, List<StopReplicaTopicState> topicStates,
- boolean kraftController) {
- super(ApiKeys.STOP_REPLICA, version, controllerId, controllerEpoch, brokerEpoch, kraftController);
- this.deletePartitions = deletePartitions;
- this.topicStates = topicStates;
- }
-
- public StopReplicaRequest build(short version) {
- StopReplicaRequestData data = new StopReplicaRequestData()
- .setControllerId(controllerId)
- .setControllerEpoch(controllerEpoch)
- .setBrokerEpoch(brokerEpoch);
-
- if (version >= 4) {
- data.setIsKRaftController(kraftController);
- }
-
- if (version >= 3) {
- data.setTopicStates(topicStates);
- } else if (version >= 1) {
- data.setDeletePartitions(deletePartitions);
- List<StopReplicaTopicV1> topics = topicStates.stream().map(topic ->
- new StopReplicaTopicV1()
- .setName(topic.topicName())
- .setPartitionIndexes(topic.partitionStates().stream()
- .map(StopReplicaPartitionState::partitionIndex)
- .collect(Collectors.toList())))
- .collect(Collectors.toList());
- data.setTopics(topics);
- } else {
- data.setDeletePartitions(deletePartitions);
- List<StopReplicaPartitionV0> partitions = topicStates.stream().flatMap(topic ->
- topic.partitionStates().stream().map(partition ->
- new StopReplicaPartitionV0()
- .setTopicName(topic.topicName())
- .setPartitionIndex(partition.partitionIndex())))
- .collect(Collectors.toList());
- data.setUngroupedPartitions(partitions);
- }
-
- return new StopReplicaRequest(data, version);
- }
-
- @Override
- public String toString() {
- return "(type=StopReplicaRequest" +
- ", controllerId=" + controllerId +
- ", controllerEpoch=" + controllerEpoch +
- ", brokerEpoch=" + brokerEpoch +
- ", deletePartitions=" + deletePartitions +
- ", topicStates=" + topicStates.stream().map(StopReplicaTopicState::toString).collect(Collectors.joining(",")) +
- ")";
- }
- }
-
- private final StopReplicaRequestData data;
-
- private StopReplicaRequest(StopReplicaRequestData data, short version) {
- super(ApiKeys.STOP_REPLICA, version);
- this.data = data;
- }
-
- @Override
- public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- Errors error = Errors.forException(e);
-
- StopReplicaResponseData data = new StopReplicaResponseData();
- data.setErrorCode(error.code());
-
- List<StopReplicaPartitionError> partitions = new ArrayList<>();
- for (StopReplicaTopicState topic : topicStates()) {
- for (StopReplicaPartitionState partition : topic.partitionStates()) {
- partitions.add(new StopReplicaPartitionError()
- .setTopicName(topic.topicName())
- .setPartitionIndex(partition.partitionIndex())
- .setErrorCode(error.code()));
- }
- }
- data.setPartitionErrors(partitions);
-
- return new StopReplicaResponse(data);
- }
-
- /**
- * Note that this method has allocation overhead per iterated element, so callers should copy the result into
- * another collection if they need to iterate more than once.
- *
- * Implementation note: we should strive to avoid allocation overhead per element, see
- * `UpdateMetadataRequest.partitionStates()` for the preferred approach. That's not possible in this case and
- * StopReplicaRequest should be relatively rare in comparison to other request types.
- */
- public Iterable<StopReplicaTopicState> topicStates() {
- if (version() < 1) {
- Map<String, StopReplicaTopicState> topicStates = new HashMap<>();
- for (StopReplicaPartitionV0 partition : data.ungroupedPartitions()) {
- StopReplicaTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
- topic -> new StopReplicaTopicState().setTopicName(topic));
- topicState.partitionStates().add(new StopReplicaPartitionState()
- .setPartitionIndex(partition.partitionIndex())
- .setDeletePartition(data.deletePartitions()));
- }
- return topicStates.values();
- } else if (version() < 3) {
- return () -> new MappedIterator<>(data.topics().iterator(), topic ->
- new StopReplicaTopicState()
- .setTopicName(topic.name())
- .setPartitionStates(topic.partitionIndexes().stream()
- .map(partition -> new StopReplicaPartitionState()
- .setPartitionIndex(partition)
- .setDeletePartition(data.deletePartitions()))
- .collect(Collectors.toList())));
- } else {
- return data.topicStates();
- }
- }
-
- public Map<TopicPartition, StopReplicaPartitionState> partitionStates() {
- Map<TopicPartition, StopReplicaPartitionState> partitionStates = new HashMap<>();
-
- if (version() < 1) {
- for (StopReplicaPartitionV0 partition : data.ungroupedPartitions()) {
- partitionStates.put(
- new TopicPartition(partition.topicName(), partition.partitionIndex()),
- new StopReplicaPartitionState()
- .setPartitionIndex(partition.partitionIndex())
- .setDeletePartition(data.deletePartitions()));
- }
- } else if (version() < 3) {
- for (StopReplicaTopicV1 topic : data.topics()) {
- for (Integer partitionIndex : topic.partitionIndexes()) {
- partitionStates.put(
- new TopicPartition(topic.name(), partitionIndex),
- new StopReplicaPartitionState()
- .setPartitionIndex(partitionIndex)
- .setDeletePartition(data.deletePartitions()));
- }
- }
- } else {
- for (StopReplicaTopicState topicState : data.topicStates()) {
- for (StopReplicaPartitionState partitionState: topicState.partitionStates()) {
- partitionStates.put(
- new TopicPartition(topicState.topicName(), partitionState.partitionIndex()),
- partitionState);
- }
- }
- }
-
- return partitionStates;
- }
-
- @Override
- public int controllerId() {
- return data.controllerId();
- }
-
- @Override
- public boolean isKRaftController() {
- return data.isKRaftController();
- }
-
- @Override
- public int controllerEpoch() {
- return data.controllerEpoch();
- }
-
- @Override
- public long brokerEpoch() {
- return data.brokerEpoch();
- }
-
- public static StopReplicaRequest parse(ByteBuffer buffer, short version) {
- return new StopReplicaRequest(new StopReplicaRequestData(new ByteBufferAccessor(buffer), version), version);
- }
-
- @Override
- public StopReplicaRequestData data() {
- return data;
- }
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
deleted file mode 100644
index cb66f4915d1..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.message.StopReplicaResponseData;
-import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-public class StopReplicaResponse extends AbstractResponse {
-
- /**
- * Possible error code:
- * - {@link Errors#STALE_CONTROLLER_EPOCH}
- * - {@link Errors#STALE_BROKER_EPOCH}
- * - {@link Errors#FENCED_LEADER_EPOCH}
- * - {@link Errors#KAFKA_STORAGE_ERROR}
- */
- private final StopReplicaResponseData data;
-
- public StopReplicaResponse(StopReplicaResponseData data) {
- super(ApiKeys.STOP_REPLICA);
- this.data = data;
- }
-
- public List<StopReplicaPartitionError> partitionErrors() {
- return data.partitionErrors();
- }
-
- public Errors error() {
- return Errors.forCode(data.errorCode());
- }
-
- @Override
- public Map<Errors, Integer> errorCounts() {
- if (data.errorCode() != Errors.NONE.code())
- // Minor optimization since the top-level error applies to all partitions
- return Collections.singletonMap(error(), data.partitionErrors().size() + 1);
- Map<Errors, Integer> errors = errorCounts(data.partitionErrors().stream().map(p -> Errors.forCode(p.errorCode())));
- updateErrorCounts(errors, Errors.forCode(data.errorCode())); // top level error
- return errors;
- }
-
- public static StopReplicaResponse parse(ByteBuffer buffer, short version) {
- return new StopReplicaResponse(new StopReplicaResponseData(new ByteBufferAccessor(buffer), version));
- }
-
- @Override
- public int throttleTimeMs() {
- return DEFAULT_THROTTLE_TIME;
- }
-
- @Override
- public void maybeSetThrottleTimeMs(int throttleTimeMs) {
- // Not supported by the response schema
- }
-
- @Override
- public StopReplicaResponseData data() {
- return data;
- }
-
- @Override
- public String toString() {
- return data.toString();
- }
-
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
deleted file mode 100644
index 15a4dfff1a6..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.errors.UnsupportedVersionException;
-import org.apache.kafka.common.message.UpdateMetadataRequestData;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataTopicState;
-import org.apache.kafka.common.message.UpdateMetadataResponseData;
-import org.apache.kafka.common.network.ListenerName;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.security.auth.SecurityProtocol;
-import org.apache.kafka.common.utils.FlattenedIterator;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static java.util.Collections.singletonList;
-
-public final class UpdateMetadataRequest extends AbstractControlRequest {
-
- public static class Builder extends AbstractControlRequest.Builder<UpdateMetadataRequest> {
- private final List<UpdateMetadataPartitionState> partitionStates;
- private final List<UpdateMetadataBroker> liveBrokers;
- private final Map<String, Uuid> topicIds;
- private final Type updateType;
-
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- List<UpdateMetadataPartitionState> partitionStates, List<UpdateMetadataBroker> liveBrokers,
- Map<String, Uuid> topicIds) {
- this(version, controllerId, controllerEpoch, brokerEpoch, partitionStates,
- liveBrokers, topicIds, false, Type.UNKNOWN);
- }
-
- public Builder(short version, int controllerId, int controllerEpoch, long brokerEpoch,
- List<UpdateMetadataPartitionState> partitionStates, List<UpdateMetadataBroker> liveBrokers,
- Map<String, Uuid> topicIds, boolean kraftController, Type updateType) {
- super(ApiKeys.UPDATE_METADATA, version, controllerId, controllerEpoch, brokerEpoch, kraftController);
- this.partitionStates = partitionStates;
- this.liveBrokers = liveBrokers;
- this.topicIds = topicIds;
-
- if (version >= 8) {
- this.updateType = updateType;
- } else {
- this.updateType = Type.UNKNOWN;
- }
- }
-
- @Override
- public UpdateMetadataRequest build(short version) {
- if (version < 3) {
- for (UpdateMetadataBroker broker : liveBrokers) {
- if (version == 0) {
- if (broker.endpoints().size() != 1)
- throw new UnsupportedVersionException("UpdateMetadataRequest v0 requires a single endpoint");
- if (broker.endpoints().get(0).securityProtocol() != SecurityProtocol.PLAINTEXT.id)
- throw new UnsupportedVersionException("UpdateMetadataRequest v0 only handles PLAINTEXT endpoints");
- // Don't null out `endpoints` since it's ignored by the generated code if version >= 1
- UpdateMetadataEndpoint endpoint = broker.endpoints().get(0);
- broker.setV0Host(endpoint.host());
- broker.setV0Port(endpoint.port());
- } else {
- if (broker.endpoints().stream().anyMatch(endpoint -> !endpoint.listener().isEmpty() &&
- !endpoint.listener().equals(listenerNameFromSecurityProtocol(endpoint)))) {
- throw new UnsupportedVersionException("UpdateMetadataRequest v0-v3 does not support custom " +
- "listeners, request version: " + version + ", endpoints: " + broker.endpoints());
- }
- }
- }
- }
-
- UpdateMetadataRequestData data = new UpdateMetadataRequestData()
- .setControllerId(controllerId)
- .setControllerEpoch(controllerEpoch)
- .setBrokerEpoch(brokerEpoch)
- .setLiveBrokers(liveBrokers);
-
- if (version >= 8) {
- data.setIsKRaftController(kraftController);
- data.setType(updateType.toByte());
- }
-
- if (version >= 5) {
- Map<String, UpdateMetadataTopicState> topicStatesMap = groupByTopic(topicIds, partitionStates);
- data.setTopicStates(new ArrayList<>(topicStatesMap.values()));
- } else {
- data.setUngroupedPartitionStates(partitionStates);
- }
-
- return new UpdateMetadataRequest(data, version);
- }
-
- private static Map<String, UpdateMetadataTopicState> groupByTopic(Map<String, Uuid> topicIds, List<UpdateMetadataPartitionState> partitionStates) {
- Map<String, UpdateMetadataTopicState> topicStates = new HashMap<>();
- for (UpdateMetadataPartitionState partition : partitionStates) {
- // We don't null out the topic name in UpdateMetadataPartitionState since it's ignored by the generated
- // code if version >= 5
- UpdateMetadataTopicState topicState = topicStates.computeIfAbsent(partition.topicName(),
- t -> new UpdateMetadataTopicState()
- .setTopicName(partition.topicName())
- .setTopicId(topicIds.getOrDefault(partition.topicName(), Uuid.ZERO_UUID))
-
- );
- topicState.partitionStates().add(partition);
- }
- return topicStates;
- }
-
- @Override
- public String toString() {
- return "(type: UpdateMetadataRequest=" +
- ", controllerId=" + controllerId +
- ", controllerEpoch=" + controllerEpoch +
- ", kraftController=" + kraftController +
- ", type=" + updateType +
- ", brokerEpoch=" + brokerEpoch +
- ", partitionStates=" + partitionStates +
- ", liveBrokers=" + liveBrokers.stream().map(UpdateMetadataBroker::toString).collect(Collectors.joining(", ")) +
- ")";
- }
- }
-
- private final UpdateMetadataRequestData data;
-
- public UpdateMetadataRequest(UpdateMetadataRequestData data, short version) {
- super(ApiKeys.UPDATE_METADATA, version);
- this.data = data;
- // Do this from the constructor to make it thread-safe (even though it's only needed when some methods are called)
- normalize();
- }
-
- private void normalize() {
- // Version 0 only supported a single host and port and the protocol was always plaintext
- // Version 1 added support for multiple endpoints, each with its own security protocol
- // Version 2 added support for rack
- // Version 3 added support for listener name, which we can infer from the security protocol for older versions
- if (version() < 3) {
- for (UpdateMetadataBroker liveBroker : data.liveBrokers()) {
- // Set endpoints so that callers can rely on it always being present
- if (version() == 0 && liveBroker.endpoints().isEmpty()) {
- SecurityProtocol securityProtocol = SecurityProtocol.PLAINTEXT;
- liveBroker.setEndpoints(singletonList(new UpdateMetadataEndpoint()
- .setHost(liveBroker.v0Host())
- .setPort(liveBroker.v0Port())
- .setSecurityProtocol(securityProtocol.id)
- .setListener(ListenerName.forSecurityProtocol(securityProtocol).value())));
- } else {
- for (UpdateMetadataEndpoint endpoint : liveBroker.endpoints()) {
- // Set listener so that callers can rely on it always being present
- if (endpoint.listener().isEmpty())
- endpoint.setListener(listenerNameFromSecurityProtocol(endpoint));
- }
- }
- }
- }
-
- if (version() >= 5) {
- for (UpdateMetadataTopicState topicState : data.topicStates()) {
- for (UpdateMetadataPartitionState partitionState : topicState.partitionStates()) {
- // Set the topic name so that we can always present the ungrouped view to callers
- partitionState.setTopicName(topicState.topicName());
- }
- }
- }
- }
-
- private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint endpoint) {
- SecurityProtocol securityProtocol = SecurityProtocol.forId(endpoint.securityProtocol());
- return ListenerName.forSecurityProtocol(securityProtocol).value();
- }
-
- @Override
- public int controllerId() {
- return data.controllerId();
- }
-
- @Override
- public boolean isKRaftController() {
- return data.isKRaftController();
- }
-
- public Type updateType() {
- return Type.fromByte(data.type());
- }
-
- @Override
- public int controllerEpoch() {
- return data.controllerEpoch();
- }
-
- @Override
- public long brokerEpoch() {
- return data.brokerEpoch();
- }
-
- @Override
- public UpdateMetadataResponse getErrorResponse(int throttleTimeMs, Throwable e) {
- UpdateMetadataResponseData data = new UpdateMetadataResponseData()
- .setErrorCode(Errors.forException(e).code());
- return new UpdateMetadataResponse(data);
- }
-
- public Iterable<UpdateMetadataPartitionState> partitionStates() {
- if (version() >= 5) {
- return () -> new FlattenedIterator<>(data.topicStates().iterator(),
- topicState -> topicState.partitionStates().iterator());
- }
- return data.ungroupedPartitionStates();
- }
-
- public List<UpdateMetadataTopicState> topicStates() {
- if (version() >= 5) {
- return data.topicStates();
- }
- return Collections.emptyList();
- }
-
- public List<UpdateMetadataBroker> liveBrokers() {
- return data.liveBrokers();
- }
-
- @Override
- public UpdateMetadataRequestData data() {
- return data;
- }
-
- public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) {
- return new UpdateMetadataRequest(new UpdateMetadataRequestData(new ByteBufferAccessor(buffer), version), version);
- }
-}
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
deleted file mode 100644
index d5960d7cbb9..00000000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.message.UpdateMetadataResponseData;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.ByteBufferAccessor;
-import org.apache.kafka.common.protocol.Errors;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-
-public class UpdateMetadataResponse extends AbstractResponse {
-
- private final UpdateMetadataResponseData data;
-
- public UpdateMetadataResponse(UpdateMetadataResponseData data) {
- super(ApiKeys.UPDATE_METADATA);
- this.data = data;
- }
-
- public Errors error() {
- return Errors.forCode(data.errorCode());
- }
-
- @Override
- public Map<Errors, Integer> errorCounts() {
- return errorCounts(error());
- }
-
- @Override
- public int throttleTimeMs() {
- return DEFAULT_THROTTLE_TIME;
- }
-
- @Override
- public void maybeSetThrottleTimeMs(int throttleTimeMs) {
- // Not supported by the response schema
- }
-
- public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) {
- return new UpdateMetadataResponse(new UpdateMetadataResponseData(new ByteBufferAccessor(buffer), version));
- }
-
- @Override
- public UpdateMetadataResponseData data() {
- return data;
- }
-
-}
diff --git a/clients/src/main/resources/common/message/AlterPartitionRequest.json b/clients/src/main/resources/common/message/AlterPartitionRequest.json
index 89b5dbfdd95..954b98578af 100644
--- a/clients/src/main/resources/common/message/AlterPartitionRequest.json
+++ b/clients/src/main/resources/common/message/AlterPartitionRequest.json
@@ -54,7 +54,7 @@
{ "name": "LeaderRecoveryState", "type": "int8", "versions": "1+", "default": "0",
"about": "1 if the partition is recovering from an unclean leader election; 0 otherwise." },
{ "name": "PartitionEpoch", "type": "int32", "versions": "0+",
- "about": "The expected epoch of the partition which is being updated. For legacy cluster this is the ZkVersion in the LeaderAndIsr request." }
+ "about": "The expected epoch of the partition which is being updated." }
]}
]}
]
diff --git a/clients/src/main/resources/common/message/AlterPartitionResponse.json b/clients/src/main/resources/common/message/AlterPartitionResponse.json
index 13b26abeb3a..0df8f8c9232 100644
--- a/clients/src/main/resources/common/message/AlterPartitionResponse.json
+++ b/clients/src/main/resources/common/message/AlterPartitionResponse.json
@@ -51,7 +51,7 @@
{ "name": "LeaderRecoveryState", "type": "int8", "versions": "1+", "default": "0", "ignorable": true,
"about": "1 if the partition is recovering from an unclean leader election; 0 otherwise." },
{ "name": "PartitionEpoch", "type": "int32", "versions": "0+",
- "about": "The current epoch for the partition for KRaft controllers. The current ZK version for the legacy controllers." }
+ "about": "The current epoch for the partition for KRaft controllers." }
]}
]}
]
diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
index 49561f7b6e7..6ddff583af5 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json
@@ -16,21 +16,7 @@
{
"apiKey": 7,
"type": "request",
- "listeners": ["zkBroker", "controller"],
"name": "ControlledShutdownRequest",
- // Version 0 of ControlledShutdownRequest has a non-standard request header
- // which does not include clientId. Version 1 and later use the standard
- // request header.
- //
- // Version 1 is the same as version 0.
- //
- // Version 2 adds BrokerEpoch.
- "validVersions": "0-3",
- "flexibleVersions": "3+",
- "fields": [
- { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The id of the broker for which controlled shutdown has been requested." },
- { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true,
- "about": "The broker epoch." }
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/ControlledShutdownResponse.json b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
index 27feb1b69b0..f7daf5c0ca0 100644
--- a/clients/src/main/resources/common/message/ControlledShutdownResponse.json
+++ b/clients/src/main/resources/common/message/ControlledShutdownResponse.json
@@ -17,18 +17,6 @@
"apiKey": 7,
"type": "response",
"name": "ControlledShutdownResponse",
- // Versions 1 and 2 are the same as version 0.
- "validVersions": "0-3",
- "flexibleVersions": "3+",
- "fields": [
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The top-level error code." },
- { "name": "RemainingPartitions", "type": "[]RemainingPartition", "versions": "0+",
- "about": "The partitions that the broker still leads.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0+", "mapKey": true, "entityType": "topicName",
- "about": "The name of the topic." },
- { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true,
- "about": "The index of the partition." }
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
index ec0219a233a..30ebe4d2961 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json
@@ -16,84 +16,7 @@
{
"apiKey": 4,
"type": "request",
- "listeners": ["zkBroker"],
"name": "LeaderAndIsrRequest",
- // Version 1 adds IsNew.
- //
- // Version 2 adds broker epoch and reorganizes the partitions by topic.
- //
- // Version 3 adds AddingReplicas and RemovingReplicas.
- //
- // Version 4 is the first flexible version.
- //
- // Version 5 adds Topic ID and Type to the TopicStates, as described in KIP-516.
- //
- // Version 6 adds LeaderRecoveryState as described in KIP-704.
- //
- // Version 7 adds KRaft Controller ID field as part of KIP-866
- "validVersions": "0-7",
- "flexibleVersions": "4+",
- "fields": [
- { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The current controller ID." },
- { "name": "isKRaftController", "type": "bool", "versions": "7+", "default": "false",
- "about": "If KRaft controller id is used during migration. See KIP-866." },
- { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
- "about": "The current controller epoch." },
- { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "ignorable": true, "default": "-1",
- "about": "The current broker epoch." },
- { "name": "Type", "type": "int8", "versions": "5+",
- "about": "The type that indicates whether all topics are included in the request."},
- { "name": "UngroupedPartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "0-1",
- "about": "The state of each partition, in a v0 or v1 message." },
- // In v0 or v1 requests, each partition is listed alongside its topic name.
- // In v2+ requests, partitions are organized by topic, so that each topic name
- // only needs to be listed once.
- { "name": "TopicStates", "type": "[]LeaderAndIsrTopicState", "versions": "2+",
- "about": "Each topic.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "2+", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "TopicId", "type": "uuid", "versions": "5+", "ignorable": true,
- "about": "The unique topic ID." },
- { "name": "PartitionStates", "type": "[]LeaderAndIsrPartitionState", "versions": "2+",
- "about": "The state of each partition." }
- ]},
- { "name": "LiveLeaders", "type": "[]LeaderAndIsrLiveLeader", "versions": "0+",
- "about": "The current live leaders.", "fields": [
- { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The leader's broker ID." },
- { "name": "HostName", "type": "string", "versions": "0+",
- "about": "The leader's hostname." },
- { "name": "Port", "type": "int32", "versions": "0+",
- "about": "The leader's port." }
- ]}
- ],
- "commonStructs": [
- { "name": "LeaderAndIsrPartitionState", "versions": "0+", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0-1", "entityType": "topicName", "ignorable": true,
- "about": "The topic name. This is only present in v0 or v1." },
- { "name": "PartitionIndex", "type": "int32", "versions": "0+",
- "about": "The partition index." },
- { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
- "about": "The controller epoch." },
- { "name": "Leader", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The broker ID of the leader." },
- { "name": "LeaderEpoch", "type": "int32", "versions": "0+",
- "about": "The leader epoch." },
- { "name": "Isr", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
- "about": "The in-sync replica IDs." },
- { "name": "PartitionEpoch", "type": "int32", "versions": "0+",
- "about": "The current epoch for the partition. The epoch is a monotonically increasing value which is incremented after every partition change. (Since the LeaderAndIsr request is only used by the legacy controller, this corresponds to the zkVersion)." },
- { "name": "Replicas", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
- "about": "The replica IDs." },
- { "name": "AddingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true, "entityType": "brokerId",
- "about": "The replica IDs that we are adding this partition to, or null if no replicas are being added." },
- { "name": "RemovingReplicas", "type": "[]int32", "versions": "3+", "ignorable": true, "entityType": "brokerId",
- "about": "The replica IDs that we are removing this partition from, or null if no replicas are being removed." },
- { "name": "IsNew", "type": "bool", "versions": "1+", "default": "false", "ignorable": true,
- "about": "Whether the replica should have existed on the broker or not." },
- { "name": "LeaderRecoveryState", "type": "int8", "versions": "6+", "default": "0",
- "about": "1 if the partition is recovering from an unclean leader election; 0 otherwise." }
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
index d97f55bf4b9..59e04fb200b 100644
--- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
+++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json
@@ -17,39 +17,6 @@
"apiKey": 4,
"type": "response",
"name": "LeaderAndIsrResponse",
- // Version 1 adds KAFKA_STORAGE_ERROR as a valid error code.
- //
- // Version 2 is the same as version 1.
- //
- // Version 3 is the same as version 2.
- //
- // Version 4 is the first flexible version.
- //
- // Version 5 removes TopicName and replaces it with TopicId and reorganizes
- // the partitions by topic, as described by KIP-516.
- "validVersions": "0-7",
- "flexibleVersions": "4+",
- "fields": [
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The error code, or 0 if there was no error." },
- { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "0-4",
- "about": "Each partition in v0 to v4 message."},
- { "name": "Topics", "type": "[]LeaderAndIsrTopicError", "versions": "5+",
- "about": "Each topic.", "fields": [
- { "name": "TopicId", "type": "uuid", "versions": "5+", "mapKey": true,
- "about": "The unique topic ID." },
- { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "5+",
- "about": "Each partition."}
- ]}
- ],
- "commonStructs": [
- { "name": "LeaderAndIsrPartitionError", "versions": "0+", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName", "ignorable": true,
- "about": "The topic name."},
- { "name": "PartitionIndex", "type": "int32", "versions": "0+",
- "about": "The partition index." },
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The partition error code, or 0 if there was no error." }
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json
index fbf4e2ca9f4..50589768fe1 100644
--- a/clients/src/main/resources/common/message/RequestHeader.json
+++ b/clients/src/main/resources/common/message/RequestHeader.json
@@ -16,12 +16,14 @@
{
"type": "header",
"name": "RequestHeader",
+ // Version 0 was removed in Apache Kafka 4.0, Version 1 is the new baseline.
+ //
// Version 0 of the RequestHeader is only used by v0 of ControlledShutdownRequest.
//
// Version 1 is the first version with ClientId.
//
// Version 2 is the first flexible version.
- "validVersions": "0-2",
+ "validVersions": "1-2",
"flexibleVersions": "2+",
"fields": [
{ "name": "RequestApiKey", "type": "int16", "versions": "0+",
@@ -36,7 +38,7 @@
// ApiVersionsRequest, even if it is from a newer version.
// Since the client is sending the ApiVersionsRequest in order to discover what
// versions are supported, the client does not know the best version to use.
- { "name": "ClientId", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true,
- "flexibleVersions": "none", "about": "The client ID string." }
+ { "name": "ClientId", "type": "string", "versions": "1+", "nullableVersions": "1+", "flexibleVersions": "none",
+ "about": "The client ID string." }
]
}
diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json
index c5f0d4c8124..62d96f61440 100644
--- a/clients/src/main/resources/common/message/StopReplicaRequest.json
+++ b/clients/src/main/resources/common/message/StopReplicaRequest.json
@@ -16,56 +16,7 @@
{
"apiKey": 5,
"type": "request",
- "listeners": ["zkBroker"],
"name": "StopReplicaRequest",
- // Version 1 adds the broker epoch and reorganizes the partitions to be stored
- // per topic.
- //
- // Version 2 is the first flexible version.
- //
- // Version 3 adds the leader epoch per partition (KIP-570).
- //
- // Version 4 adds KRaft Controller ID field as part of KIP-866
- "validVersions": "0-4",
- "flexibleVersions": "2+",
- "fields": [
- { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The controller id." },
- { "name": "isKRaftController", "type": "bool", "versions": "4+", "default": "false",
- "about": "If KRaft controller id is used during migration. See KIP-866." },
- { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
- "about": "The controller epoch." },
- { "name": "BrokerEpoch", "type": "int64", "versions": "1+", "default": "-1", "ignorable": true,
- "about": "The broker epoch." },
- { "name": "DeletePartitions", "type": "bool", "versions": "0-2",
- "about": "Whether these partitions should be deleted." },
- { "name": "UngroupedPartitions", "type": "[]StopReplicaPartitionV0", "versions": "0",
- "about": "The partitions to stop.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "PartitionIndex", "type": "int32", "versions": "0",
- "about": "The partition index." }
- ]},
- { "name": "Topics", "type": "[]StopReplicaTopicV1", "versions": "1-2",
- "about": "The topics to stop.", "fields": [
- { "name": "Name", "type": "string", "versions": "1-2", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "PartitionIndexes", "type": "[]int32", "versions": "1-2",
- "about": "The partition indexes." }
- ]},
- { "name": "TopicStates", "type": "[]StopReplicaTopicState", "versions": "3+",
- "about": "Each topic.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "3+", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "PartitionStates", "type": "[]StopReplicaPartitionState", "versions": "3+",
- "about": "The state of each partition.", "fields": [
- { "name": "PartitionIndex", "type": "int32", "versions": "3+",
- "about": "The partition index." },
- { "name": "LeaderEpoch", "type": "int32", "versions": "3+", "default": "-1",
- "about": "The leader epoch." },
- { "name": "DeletePartition", "type": "bool", "versions": "3+",
- "about": "Whether this partition should be deleted." }
- ]}
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/StopReplicaResponse.json b/clients/src/main/resources/common/message/StopReplicaResponse.json
index 35551c47e1c..43de9c688b7 100644
--- a/clients/src/main/resources/common/message/StopReplicaResponse.json
+++ b/clients/src/main/resources/common/message/StopReplicaResponse.json
@@ -17,24 +17,6 @@
"apiKey": 5,
"type": "response",
"name": "StopReplicaResponse",
- // Version 1 is the same as version 0.
- //
- // Version 2 is the first flexible version.
- //
- // Version 3 returns FENCED_LEADER_EPOCH if the epoch of the leader is stale (KIP-570).
- "validVersions": "0-4",
- "flexibleVersions": "2+",
- "fields": [
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The top-level error code, or 0 if there was no top-level error." },
- { "name": "PartitionErrors", "type": "[]StopReplicaPartitionError", "versions": "0+",
- "about": "The responses for each partition.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0+", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "PartitionIndex", "type": "int32", "versions": "0+",
- "about": "The partition index." },
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The partition error code, or 0 if there was no partition error." }
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
index c4ab4c44232..68beeb7a8bf 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json
@@ -16,90 +16,7 @@
{
"apiKey": 6,
"type": "request",
- "listeners": ["zkBroker"],
"name": "UpdateMetadataRequest",
- // Version 1 allows specifying multiple endpoints for each broker.
- //
- // Version 2 adds the rack.
- //
- // Version 3 adds the listener name.
- //
- // Version 4 adds the offline replica list.
- //
- // Version 5 adds the broker epoch field and normalizes partitions by topic.
- //
- // Version 7 adds topicId
- //
- // Version 8 adds KRaft Controller ID field as part of KIP-866
- "validVersions": "0-8",
- "flexibleVersions": "6+",
- "fields": [
- { "name": "ControllerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The controller id." },
- { "name": "isKRaftController", "type": "bool", "versions": "8+", "default": "false",
- "about": "If KRaft controller id is used during migration. See KIP-866." },
- { "name": "Type", "type": "int8", "versions": "8+",
- "default": 0, "tag": 0, "taggedVersions": "8+",
- "about": "Indicates if this request is a Full metadata snapshot (2), Incremental (1), or Unknown (0). Using during ZK migration, see KIP-866."},
- { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
- "about": "The controller epoch." },
- { "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1",
- "about": "The broker epoch." },
- { "name": "UngroupedPartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "0-4",
- "about": "In older versions of this RPC, each partition that we would like to update." },
- { "name": "TopicStates", "type": "[]UpdateMetadataTopicState", "versions": "5+",
- "about": "In newer versions of this RPC, each topic that we would like to update.", "fields": [
- { "name": "TopicName", "type": "string", "versions": "5+", "entityType": "topicName",
- "about": "The topic name." },
- { "name": "TopicId", "type": "uuid", "versions": "7+", "ignorable": true, "about": "The topic id."},
- { "name": "PartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "5+",
- "about": "The partition that we would like to update." }
- ]},
- { "name": "LiveBrokers", "type": "[]UpdateMetadataBroker", "versions": "0+",
- "about": "The brokers that we know about.", "fields": [
- { "name": "Id", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The broker id." },
- // Version 0 of the protocol only allowed specifying a single host and
- // port per broker, rather than an array of endpoints.
- { "name": "V0Host", "type": "string", "versions": "0", "ignorable": true,
- "about": "The broker hostname." },
- { "name": "V0Port", "type": "int32", "versions": "0", "ignorable": true,
- "about": "The broker port." },
- { "name": "Endpoints", "type": "[]UpdateMetadataEndpoint", "versions": "1+", "ignorable": true,
- "about": "The broker endpoints.", "fields": [
- { "name": "Port", "type": "int32", "versions": "1+",
- "about": "The port of this endpoint." },
- { "name": "Host", "type": "string", "versions": "1+",
- "about": "The hostname of this endpoint." },
- { "name": "Listener", "type": "string", "versions": "3+", "ignorable": true,
- "about": "The listener name." },
- { "name": "SecurityProtocol", "type": "int16", "versions": "1+",
- "about": "The security protocol type." }
- ]},
- { "name": "Rack", "type": "string", "versions": "2+", "nullableVersions": "0+", "ignorable": true,
- "about": "The rack which this broker belongs to." }
- ]}
- ],
- "commonStructs": [
- { "name": "UpdateMetadataPartitionState", "versions": "0+", "fields": [
- { "name": "TopicName", "type": "string", "versions": "0-4", "entityType": "topicName", "ignorable": true,
- "about": "In older versions of this RPC, the topic name." },
- { "name": "PartitionIndex", "type": "int32", "versions": "0+",
- "about": "The partition index." },
- { "name": "ControllerEpoch", "type": "int32", "versions": "0+",
- "about": "The controller epoch." },
- { "name": "Leader", "type": "int32", "versions": "0+", "entityType": "brokerId",
- "about": "The ID of the broker which is the current partition leader." },
- { "name": "LeaderEpoch", "type": "int32", "versions": "0+",
- "about": "The leader epoch of this partition." },
- { "name": "Isr", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
- "about": "The brokers which are in the ISR for this partition." },
- { "name": "ZkVersion", "type": "int32", "versions": "0+",
- "about": "The Zookeeper version." },
- { "name": "Replicas", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
- "about": "All the replicas of this partition." },
- { "name": "OfflineReplicas", "type": "[]int32", "versions": "4+", "entityType": "brokerId", "ignorable": true,
- "about": "The replicas of this partition which are offline." }
- ]}
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/UpdateMetadataResponse.json b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
index 4e091e50101..f3cef2becb3 100644
--- a/clients/src/main/resources/common/message/UpdateMetadataResponse.json
+++ b/clients/src/main/resources/common/message/UpdateMetadataResponse.json
@@ -17,11 +17,6 @@
"apiKey": 6,
"type": "response",
"name": "UpdateMetadataResponse",
- // Versions 1, 2, 3, 4, and 5 are the same as version 0
- "validVersions": "0-8",
- "flexibleVersions": "6+",
- "fields": [
- { "name": "ErrorCode", "type": "int16", "versions": "0+",
- "about": "The error code, or 0 if there was no error." }
- ]
+ // This request was removed in Apache Kafka 4.0.
+ "validVersions": "none"
}
diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
index dc4b5be42a7..933d009b558 100644
--- a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
+++ b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json
@@ -18,8 +18,10 @@
"type": "request",
"listeners": ["zkBroker", "broker"],
"name": "WriteTxnMarkersRequest",
+ // Version 0 was removed in Apache Kafka 4.0, Version 1 is the new baseline.
+ //
// Version 1 enables flexible versions.
- "validVersions": "0-1",
+ "validVersions": "1",
"flexibleVersions": "1+",
"fields": [
{ "name": "Markers", "type": "[]WritableTxnMarker", "versions": "0+",
diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json b/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
index 59b8d66f2d4..72c5e30c368 100644
--- a/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
+++ b/clients/src/main/resources/common/message/WriteTxnMarkersResponse.json
@@ -17,7 +17,9 @@
"apiKey": 27,
"type": "response",
"name": "WriteTxnMarkersResponse",
- "validVersions": "0-1",
+ "validVersions": "1",
+ // Version 0 was removed in Apache Kafka 4.0, Version 1 is the new baseline.
+ //
// Version 1 enables flexible versions.
"flexibleVersions": "1+",
"fields": [
diff --git a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
index ad1f614509f..8b22bc7963d 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java
@@ -80,6 +80,8 @@ public class NodeApiVersionsTest {
bld.append(prefix);
if (apiKey == ApiKeys.DELETE_TOPICS) {
bld.append("DeleteTopics(20): 10000 to 10001 [unusable: node too new]");
+ } else if (!apiKey.hasValidVersion()) {
+ bld.append(apiKey.name + "(" + apiKey.id + "): 0 to -1 [unusable: node too new]");
} else {
bld.append(apiKey.name).append("(").
append(apiKey.id).append("): ");
diff --git a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
index 4728e67f719..57bca731177 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/ApiMessageTypeTest.java
@@ -18,6 +18,7 @@
package org.apache.kafka.common.message;
import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.types.Schema;
import org.junit.jupiter.api.Test;
@@ -30,6 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@ -59,22 +61,28 @@ public class ApiMessageTypeTest {
Set<Short> ids = new HashSet<>();
Set<String> requestNames = new HashSet<>();
Set<String> responseNames = new HashSet<>();
+ int apiKeysWithNoValidVersionCount = 0;
for (ApiMessageType type : ApiMessageType.values()) {
assertFalse(ids.contains(type.apiKey()),
"found two ApiMessageType objects with id " + type.apiKey());
ids.add(type.apiKey());
- String requestName = type.newRequest().getClass().getSimpleName();
- assertFalse(requestNames.contains(requestName),
- "found two ApiMessageType objects with requestName " + requestName);
- requestNames.add(requestName);
- String responseName = type.newResponse().getClass().getSimpleName();
- assertFalse(responseNames.contains(responseName),
- "found two ApiMessageType objects with responseName " + responseName);
- responseNames.add(responseName);
+ ApiKeys apiKey = ApiKeys.forId(type.apiKey());
+ if (apiKey.hasValidVersion()) {
+ String requestName = type.newRequest().getClass().getSimpleName();
+ assertFalse(requestNames.contains(requestName),
+ "found two ApiMessageType objects with requestName " + requestName);
+ requestNames.add(requestName);
+ String responseName = type.newResponse().getClass().getSimpleName();
+ assertFalse(responseNames.contains(responseName),
+ "found two ApiMessageType objects with responseName " + responseName);
+ responseNames.add(responseName);
+ } else
+ ++apiKeysWithNoValidVersionCount;
}
assertEquals(ApiMessageType.values().length, ids.size());
- assertEquals(ApiMessageType.values().length, requestNames.size());
- assertEquals(ApiMessageType.values().length, responseNames.size());
+ int expectedNamesCount = ApiMessageType.values().length - apiKeysWithNoValidVersionCount;
+ assertEquals(expectedNamesCount, requestNames.size());
+ assertEquals(expectedNamesCount, responseNames.size());
}
@Test
@@ -85,12 +93,6 @@ public class ApiMessageTypeTest {
assertEquals((short) 1, ApiMessageType.PRODUCE.requestHeaderVersion((short) 1));
assertEquals((short) 0, ApiMessageType.PRODUCE.responseHeaderVersion((short) 1));
- assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.requestHeaderVersion((short) 0));
- assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.responseHeaderVersion((short) 0));
-
- assertEquals((short) 1, ApiMessageType.CONTROLLED_SHUTDOWN.requestHeaderVersion((short) 1));
- assertEquals((short) 0, ApiMessageType.CONTROLLED_SHUTDOWN.responseHeaderVersion((short) 1));
-
assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.requestHeaderVersion((short) 4));
assertEquals((short) 0, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 4));
@@ -98,6 +100,16 @@ public class ApiMessageTypeTest {
assertEquals((short) 1, ApiMessageType.CREATE_TOPICS.responseHeaderVersion((short) 5));
}
+ @Test
+ public void testHeaderVersionWithNoValidVersion() {
+ for (ApiMessageType messageType : ApiMessageType.values()) {
+ if (messageType.lowestSupportedVersion() > messageType.highestSupportedVersion(true)) {
+ assertThrows(UnsupportedVersionException.class, () -> messageType.requestHeaderVersion((short) 0));
+ assertThrows(UnsupportedVersionException.class, () -> messageType.responseHeaderVersion((short) 0));
+ }
+ }
+ }
+
@Test
public void testAllVersionsHaveSchemas() {
for (ApiMessageType type : ApiMessageType.values()) {
diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
index a64eada22d9..2786f5b62d8 100644
--- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java
@@ -409,35 +409,6 @@ public final class MessageTest {
new OffsetForLeaderEpochRequestData().setReplicaId(-2));
}
- @Test
- public void testLeaderAndIsrVersions() throws Exception {
- // Version 3 adds two new fields - AddingReplicas and RemovingReplicas
- LeaderAndIsrRequestData.LeaderAndIsrTopicState partitionStateNoAddingRemovingReplicas =
- new LeaderAndIsrRequestData.LeaderAndIsrTopicState()
- .setTopicName("topic")
- .setPartitionStates(Collections.singletonList(
- new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
- .setPartitionIndex(0)
- .setReplicas(Collections.singletonList(0))
- ));
- LeaderAndIsrRequestData.LeaderAndIsrTopicState partitionStateWithAddingRemovingReplicas =
- new LeaderAndIsrRequestData.LeaderAndIsrTopicState()
- .setTopicName("topic")
- .setPartitionStates(Collections.singletonList(
- new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
- .setPartitionIndex(0)
- .setReplicas(Collections.singletonList(0))
- .setAddingReplicas(Collections.singletonList(1))
- .setRemovingReplicas(Collections.singletonList(1))
- ));
- testAllMessageRoundTripsBetweenVersions(
- (short) 2,
- (short) 3,
- new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)),
- new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateNoAddingRemovingReplicas)));
- testAllMessageRoundTripsFromVersion((short) 3, new LeaderAndIsrRequestData().setTopicStates(Collections.singletonList(partitionStateWithAddingRemovingReplicas)));
- }
-
@Test
public void testOffsetCommitRequestVersions() throws Exception {
String groupId = "groupId";
@@ -1095,23 +1066,25 @@ public final class MessageTest {
@Test
public void testMessageVersions() {
for (ApiKeys apiKey : ApiKeys.values()) {
- Message message = null;
- try {
- message = ApiMessageType.fromApiKey(apiKey.id).newRequest();
- } catch (UnsupportedVersionException e) {
- fail("No request message spec found for API " + apiKey);
- }
- assertTrue(apiKey.latestVersion() <= message.highestSupportedVersion(),
- "Request message spec for " + apiKey + " only " + "supports versions up to " +
- message.highestSupportedVersion());
- try {
- message = ApiMessageType.fromApiKey(apiKey.id).newResponse();
- } catch (UnsupportedVersionException e) {
- fail("No response message spec found for API " + apiKey);
+ if (apiKey.hasValidVersion()) {
+ Message message = null;
+ try {
+ message = ApiMessageType.fromApiKey(apiKey.id).newRequest();
+ } catch (UnsupportedVersionException e) {
+ fail("No request message spec found for API " + apiKey);
+ }
+ assertTrue(apiKey.latestVersion() <= message.highestSupportedVersion(),
+ "Request message spec for " + apiKey + " only " + "supports versions up to " +
+ message.highestSupportedVersion());
+ try {
+ message = ApiMessageType.fromApiKey(apiKey.id).newResponse();
+ } catch (UnsupportedVersionException e) {
+ fail("No response message spec found for API " + apiKey);
+ }
+ assertTrue(apiKey.latestVersion() <= message.highestSupportedVersion(),
+ "Response message spec for " + apiKey + " only " + "supports versions up to " +
+ message.highestSupportedVersion());
}
- assertTrue(apiKey.latestVersion() <= message.highestSupportedVersion(),
- "Response message spec for " + apiKey + " only " + "supports versions up to " +
- message.highestSupportedVersion());
}
}
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java
index a4b785a908f..85ba9b46b31 100644
--- a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java
@@ -80,7 +80,7 @@ public class ApiKeysTest {
public void testApiScope() {
Set<ApiKeys> apisMissingScope = new HashSet<>();
for (ApiKeys apiKey : ApiKeys.values()) {
- if (apiKey.messageType.listeners().isEmpty()) {
+ if (apiKey.messageType.listeners().isEmpty() && apiKey.hasValidVersion()) {
apisMissingScope.add(apiKey);
}
}
@@ -88,6 +88,18 @@ public class ApiKeysTest {
"Found some APIs missing scope definition");
}
+ @Test
+ public void testHasValidVersions() {
+ var apiKeysWithNoValidVersions = Set.of(ApiKeys.LEADER_AND_ISR, ApiKeys.STOP_REPLICA, ApiKeys.UPDATE_METADATA,
+ ApiKeys.CONTROLLED_SHUTDOWN);
+ for (ApiKeys apiKey : ApiKeys.values()) {
+ if (apiKeysWithNoValidVersions.contains(apiKey))
+ assertFalse(apiKey.hasValidVersion());
+ else
+ assertTrue(apiKey.hasValidVersion());
+ }
+ }
+
@Test
public void testHtmlOnlyHaveStableApi() {
String html = ApiKeys.toHtml();
diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/ProtocolTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/ProtocolTest.java
new file mode 100644
index 00000000000..adcabba1a49
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/common/protocol/ProtocolTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.common.protocol;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class ProtocolTest {
+
+ @Test
+ public void testToHtml() {
+ var html = Protocol.toHtml();
+ assertFalse(html.isBlank());
+ assertFalse(html.contains("LeaderAndIsr"), "Removed LeaderAndIsr should not show in HTML");
+ }
+
+}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java
deleted file mode 100644
index 3f55aaf985e..00000000000
--- a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.errors.ClusterAuthorizationException;
-import org.apache.kafka.common.errors.UnsupportedVersionException;
-import org.apache.kafka.common.message.ControlledShutdownRequestData;
-import org.apache.kafka.common.protocol.Errors;
-
-import org.junit.jupiter.api.Test;
-
-import static org.apache.kafka.common.protocol.ApiKeys.CONTROLLED_SHUTDOWN;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-public class ControlledShutdownRequestTest {
-
- @Test
- public void testUnsupportedVersion() {
- ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder(
- new ControlledShutdownRequestData().setBrokerId(1),
- (short) (CONTROLLED_SHUTDOWN.latestVersion() + 1));
- assertThrows(UnsupportedVersionException.class, builder::build);
- }
-
- @Test
- public void testGetErrorResponse() {
- for (short version : CONTROLLED_SHUTDOWN.allVersions()) {
- ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder(
- new ControlledShutdownRequestData().setBrokerId(1), version);
- ControlledShutdownRequest request = builder.build();
- ControlledShutdownResponse response = request.getErrorResponse(0,
- new ClusterAuthorizationException("Not authorized"));
- assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error());
- }
- }
-
-}
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
index c0ebdbf6d52..738a06444ed 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java
@@ -32,32 +32,6 @@ import static org.mockito.Mockito.verify;
public class RequestHeaderTest {
- @Test
- public void testSerdeControlledShutdownV0() {
- // Verify that version 0 of controlled shutdown does not include the clientId field
- short apiVersion = 0;
- int correlationId = 2342;
- ByteBuffer rawBuffer = ByteBuffer.allocate(32);
- rawBuffer.putShort(ApiKeys.CONTROLLED_SHUTDOWN.id);
- rawBuffer.putShort(apiVersion);
- rawBuffer.putInt(correlationId);
- rawBuffer.flip();
-
- RequestHeader deserialized = RequestHeader.parse(rawBuffer);
- assertEquals(ApiKeys.CONTROLLED_SHUTDOWN, deserialized.apiKey());
- assertEquals(0, deserialized.apiVersion());
- assertEquals(correlationId, deserialized.correlationId());
- assertEquals("", deserialized.clientId());
- assertEquals(0, deserialized.headerVersion());
-
- ByteBuffer serializedBuffer = RequestTestUtils.serializeRequestHeader(deserialized);
-
- assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0));
- assertEquals(0, serializedBuffer.getShort(2));
- assertEquals(correlationId, serializedBuffer.getInt(4));
- assertEquals(8, serializedBuffer.limit());
- }
-
@Test
public void testRequestHeaderV1() {
short apiVersion = 1;
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 6fda62871fe..9e708a60d75 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -77,10 +77,6 @@ import org.apache.kafka.common.message.ConsumerGroupDescribeRequestData;
import org.apache.kafka.common.message.ConsumerGroupDescribeResponseData;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData;
-import org.apache.kafka.common.message.ControlledShutdownRequestData;
-import org.apache.kafka.common.message.ControlledShutdownResponseData;
-import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition;
-import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartitionCollection;
import org.apache.kafka.common.message.ControllerRegistrationRequestData;
import org.apache.kafka.common.message.ControllerRegistrationResponseData;
import org.apache.kafka.common.message.CreateAclsRequestData;
@@ -171,9 +167,6 @@ import org.apache.kafka.common.message.InitializeShareGroupStateResponseData;
import org.apache.kafka.common.message.JoinGroupRequestData;
import org.apache.kafka.common.message.JoinGroupResponseData;
import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData;
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection;
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity;
import org.apache.kafka.common.message.LeaveGroupResponseData;
import org.apache.kafka.common.message.ListClientMetricsResourcesRequestData;
@@ -230,9 +223,6 @@ import org.apache.kafka.common.message.ShareGroupDescribeRequestData;
import org.apache.kafka.common.message.ShareGroupDescribeResponseData;
import org.apache.kafka.common.message.ShareGroupHeartbeatRequestData;
import org.apache.kafka.common.message.ShareGroupHeartbeatResponseData;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionState;
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopicState;
-import org.apache.kafka.common.message.StopReplicaResponseData;
import org.apache.kafka.common.message.SyncGroupRequestData;
import org.apache.kafka.common.message.SyncGroupRequestData.SyncGroupRequestAssignment;
import org.apache.kafka.common.message.SyncGroupResponseData;
@@ -240,17 +230,12 @@ import org.apache.kafka.common.message.UnregisterBrokerRequestData;
import org.apache.kafka.common.message.UnregisterBrokerResponseData;
import org.apache.kafka.common.message.UpdateFeaturesRequestData;
import org.apache.kafka.common.message.UpdateFeaturesResponseData;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState;
-import org.apache.kafka.common.message.UpdateMetadataResponseData;
import org.apache.kafka.common.message.UpdateRaftVoterRequestData;
import org.apache.kafka.common.message.UpdateRaftVoterResponseData;
import org.apache.kafka.common.message.VoteRequestData;
import org.apache.kafka.common.message.VoteResponseData;
import org.apache.kafka.common.message.WriteShareGroupStateRequestData;
import org.apache.kafka.common.message.WriteShareGroupStateResponseData;
-import org.apache.kafka.common.network.ListenerName;
import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.ByteBufferAccessor;
@@ -296,14 +281,12 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import java.util.stream.Collectors;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.apache.kafka.common.protocol.ApiKeys.API_VERSIONS;
-import static org.apache.kafka.common.protocol.ApiKeys.CONTROLLED_SHUTDOWN;
import static org.apache.kafka.common.protocol.ApiKeys.CREATE_PARTITIONS;
import static org.apache.kafka.common.protocol.ApiKeys.CREATE_TOPICS;
import static org.apache.kafka.common.protocol.ApiKeys.DELETE_ACLS;
@@ -321,9 +304,7 @@ import static org.apache.kafka.common.protocol.ApiKeys.METADATA;
import static org.apache.kafka.common.protocol.ApiKeys.OFFSET_FETCH;
import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE;
import static org.apache.kafka.common.protocol.ApiKeys.SASL_AUTHENTICATE;
-import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA;
import static org.apache.kafka.common.protocol.ApiKeys.SYNC_GROUP;
-import static org.apache.kafka.common.protocol.ApiKeys.UPDATE_METADATA;
import static org.apache.kafka.common.protocol.ApiKeys.WRITE_TXN_MARKERS;
import static org.apache.kafka.common.requests.EndTxnRequest.LAST_STABLE_VERSION_BEFORE_TRANSACTION_V2;
import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
@@ -391,20 +372,10 @@ public class RequestResponseTest {
new NotCoordinatorException("Not Coordinator"));
checkErrorResponse(createOffsetFetchRequestWithMultipleGroups((short) 8, true),
new NotCoordinatorException("Not Coordinator"));
- // StopReplica
- for (short version : STOP_REPLICA.allVersions()) {
- checkRequest(createStopReplicaRequest(version, false));
- checkErrorResponse(createStopReplicaRequest(version, false), unknownServerException);
- }
// CreatePartitions
for (short version : CREATE_PARTITIONS.allVersions()) {
checkRequest(createCreatePartitionsRequestWithAssignments(version));
}
- // UpdateMetadata
- for (short version : UPDATE_METADATA.allVersions()) {
- checkRequest(createUpdateMetadataRequest(version, null));
- checkErrorResponse(createUpdateMetadataRequest(version, null), unknownServerException);
- }
// LeaderForEpoch
checkRequest(createLeaderEpochRequestForConsumer());
checkErrorResponse(createLeaderEpochRequestForConsumer(), unknownServerException);
@@ -929,7 +900,6 @@ public class RequestResponseTest {
assertEquals(1, createApiVersionResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createBrokerHeartbeatResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createBrokerRegistrationResponse().errorCounts().get(Errors.NONE));
- assertEquals(1, createControlledShutdownResponse().errorCounts().get(Errors.NONE));
assertEquals(2, createCreateAclsResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createCreatePartitionsResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createCreateTokenResponse().errorCounts().get(Errors.NONE));
@@ -952,8 +922,6 @@ public class RequestResponseTest {
assertEquals(1, createHeartBeatResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createIncrementalAlterConfigsResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createJoinGroupResponse(JOIN_GROUP.latestVersion()).errorCounts().get(Errors.NONE));
- assertEquals(2, createLeaderAndIsrResponse((short) 4).errorCounts().get(Errors.NONE));
- assertEquals(2, createLeaderAndIsrResponse((short) 5).errorCounts().get(Errors.NONE));
assertEquals(3, createLeaderEpochResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createLeaveGroupResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createListGroupsResponse(LIST_GROUPS.latestVersion()).errorCounts().get(Errors.NONE));
@@ -967,10 +935,8 @@ public class RequestResponseTest {
assertEquals(1, createRenewTokenResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createSaslAuthenticateResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createSaslHandshakeResponse().errorCounts().get(Errors.NONE));
- assertEquals(2, createStopReplicaResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createSyncGroupResponse(SYNC_GROUP.latestVersion()).errorCounts().get(Errors.NONE));
assertEquals(1, createTxnOffsetCommitResponse().errorCounts().get(Errors.NONE));
- assertEquals(1, createUpdateMetadataResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createWriteTxnMarkersResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createShareGroupHeartbeatResponse().errorCounts().get(Errors.NONE));
assertEquals(1, createShareGroupDescribeResponse().errorCounts().get(Errors.NONE));
@@ -984,10 +950,6 @@ public class RequestResponseTest {
case FETCH: return createFetchRequest(version);
case LIST_OFFSETS: return createListOffsetRequest(version);
case METADATA: return createMetadataRequest(version, singletonList("topic1"));
- case LEADER_AND_ISR: return createLeaderAndIsrRequest(version);
- case STOP_REPLICA: return createStopReplicaRequest(version, true);
- case UPDATE_METADATA: return createUpdateMetadataRequest(version, "rack1");
- case CONTROLLED_SHUTDOWN: return createControlledShutdownRequest(version);
case OFFSET_COMMIT: return createOffsetCommitRequest(version);
case OFFSET_FETCH: return createOffsetFetchRequest(version, true);
case FIND_COORDINATOR: return createFindCoordinatorRequest(version);
@@ -1078,10 +1040,6 @@ public class RequestResponseTest {
case FETCH: return createFetchResponse(version);
case LIST_OFFSETS: return createListOffsetResponse(version);
case METADATA: return createMetadataResponse();
- case LEADER_AND_ISR: return createLeaderAndIsrResponse(version);
- case STOP_REPLICA: return createStopReplicaResponse();
- case UPDATE_METADATA: return createUpdateMetadataResponse();
- case CONTROLLED_SHUTDOWN: return createControlledShutdownResponse();
case OFFSET_COMMIT: return createOffsetCommitResponse();
case OFFSET_FETCH: return createOffsetFetchResponse(version);
case FIND_COORDINATOR: return createFindCoordinatorResponse(version);
@@ -2521,224 +2479,6 @@ public class RequestResponseTest {
return new ProduceResponse(responseData, 0);
}
- private StopReplicaRequest createStopReplicaRequest(short version, boolean deletePartitions) {
- List<StopReplicaTopicState> topicStates = new ArrayList<>();
- StopReplicaTopicState topic1 = new StopReplicaTopicState()
- .setTopicName("topic1")
- .setPartitionStates(singletonList(new StopReplicaPartitionState()
- .setPartitionIndex(0)
- .setLeaderEpoch(1)
- .setDeletePartition(deletePartitions)));
- topicStates.add(topic1);
- StopReplicaTopicState topic2 = new StopReplicaTopicState()
- .setTopicName("topic2")
- .setPartitionStates(singletonList(new StopReplicaPartitionState()
- .setPartitionIndex(1)
- .setLeaderEpoch(2)
- .setDeletePartition(deletePartitions)));
- topicStates.add(topic2);
-
- return new StopReplicaRequest.Builder(version, 0, 1, 0,
- deletePartitions, topicStates).build(version);
- }
-
- private StopReplicaResponse createStopReplicaResponse() {
- List<StopReplicaResponseData.StopReplicaPartitionError> partitions = new ArrayList<>();
- partitions.add(new StopReplicaResponseData.StopReplicaPartitionError()
- .setTopicName("test")
- .setPartitionIndex(0)
- .setErrorCode(Errors.NONE.code()));
- return new StopReplicaResponse(new StopReplicaResponseData()
- .setErrorCode(Errors.NONE.code())
- .setPartitionErrors(partitions));
- }
-
- private ControlledShutdownRequest createControlledShutdownRequest(short version) {
- ControlledShutdownRequestData data = new ControlledShutdownRequestData()
- .setBrokerId(10)
- .setBrokerEpoch(0L);
- return new ControlledShutdownRequest.Builder(
- data,
- CONTROLLED_SHUTDOWN.latestVersion()).build(version);
- }
-
- private ControlledShutdownResponse createControlledShutdownResponse() {
- RemainingPartition p1 = new RemainingPartition()
- .setTopicName("test2")
- .setPartitionIndex(5);
- RemainingPartition p2 = new RemainingPartition()
- .setTopicName("test1")
- .setPartitionIndex(10);
- RemainingPartitionCollection pSet = new RemainingPartitionCollection();
- pSet.add(p1);
- pSet.add(p2);
- ControlledShutdownResponseData data = new ControlledShutdownResponseData()
- .setErrorCode(Errors.NONE.code())
- .setRemainingPartitions(pSet);
- return new ControlledShutdownResponse(data);
- }
-
- private LeaderAndIsrRequest createLeaderAndIsrRequest(short version) {
- List<LeaderAndIsrPartitionState> partitionStates = new ArrayList<>();
- List<Integer> isr = asList(1, 2);
- List<Integer> replicas = asList(1, 2, 3, 4);
- partitionStates.add(new LeaderAndIsrPartitionState()
- .setTopicName("topic5")
- .setPartitionIndex(105)
- .setControllerEpoch(0)
- .setLeader(2)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setPartitionEpoch(2)
- .setReplicas(replicas)
- .setIsNew(false));
- partitionStates.add(new LeaderAndIsrPartitionState()
- .setTopicName("topic5")
- .setPartitionIndex(1)
- .setControllerEpoch(1)
- .setLeader(1)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setPartitionEpoch(2)
- .setReplicas(replicas)
- .setIsNew(false));
- partitionStates.add(new LeaderAndIsrPartitionState()
- .setTopicName("topic20")
- .setPartitionIndex(1)
- .setControllerEpoch(1)
- .setLeader(0)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setPartitionEpoch(2)
- .setReplicas(replicas)
- .setIsNew(false));
-
- Set<Node> leaders = Set.of(
- new Node(0, "test0", 1223),
- new Node(1, "test1", 1223)
- );
-
- Map<String, Uuid> topicIds = new HashMap<>();
- topicIds.put("topic5", Uuid.randomUuid());
- topicIds.put("topic20", Uuid.randomUuid());
-
- return new LeaderAndIsrRequest.Builder(version, 1, 10, 0,
- partitionStates, topicIds, leaders).build();
- }
-
- private LeaderAndIsrResponse createLeaderAndIsrResponse(short version) {
- if (version < 5) {
- List<LeaderAndIsrResponseData.LeaderAndIsrPartitionError> partitions = new ArrayList<>();
- partitions.add(new LeaderAndIsrResponseData.LeaderAndIsrPartitionError()
- .setTopicName("test")
- .setPartitionIndex(0)
- .setErrorCode(Errors.NONE.code()));
- return new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
- .setErrorCode(Errors.NONE.code())
- .setPartitionErrors(partitions), version);
- } else {
- List<LeaderAndIsrResponseData.LeaderAndIsrPartitionError> partition = singletonList(
- new LeaderAndIsrResponseData.LeaderAndIsrPartitionError()
- .setPartitionIndex(0)
- .setErrorCode(Errors.NONE.code()));
- LeaderAndIsrTopicErrorCollection topics = new LeaderAndIsrTopicErrorCollection();
- topics.add(new LeaderAndIsrResponseData.LeaderAndIsrTopicError()
- .setTopicId(Uuid.randomUuid())
- .setPartitionErrors(partition));
- return new LeaderAndIsrResponse(new LeaderAndIsrResponseData()
- .setTopics(topics), version);
- }
- }
-
- private UpdateMetadataRequest createUpdateMetadataRequest(short version, String rack) {
- List<UpdateMetadataPartitionState> partitionStates = new ArrayList<>();
- List<Integer> isr = asList(1, 2);
- List<Integer> replicas = asList(1, 2, 3, 4);
- List<Integer> offlineReplicas = emptyList();
- partitionStates.add(new UpdateMetadataPartitionState()
- .setTopicName("topic5")
- .setPartitionIndex(105)
- .setControllerEpoch(0)
- .setLeader(2)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setZkVersion(2)
- .setReplicas(replicas)
- .setOfflineReplicas(offlineReplicas));
- partitionStates.add(new UpdateMetadataPartitionState()
- .setTopicName("topic5")
- .setPartitionIndex(1)
- .setControllerEpoch(1)
- .setLeader(1)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setZkVersion(2)
- .setReplicas(replicas)
- .setOfflineReplicas(offlineReplicas));
- partitionStates.add(new UpdateMetadataPartitionState()
- .setTopicName("topic20")
- .setPartitionIndex(1)
- .setControllerEpoch(1)
- .setLeader(0)
- .setLeaderEpoch(1)
- .setIsr(isr)
- .setZkVersion(2)
- .setReplicas(replicas)
- .setOfflineReplicas(offlineReplicas));
-
- Map<String, Uuid> topicIds = new HashMap<>();
- if (version > 6) {
- topicIds.put("topic5", Uuid.randomUuid());
- topicIds.put("topic20", Uuid.randomUuid());
- }
-
- SecurityProtocol plaintext = SecurityProtocol.PLAINTEXT;
- List<UpdateMetadataEndpoint> endpoints1 = new ArrayList<>();
- endpoints1.add(new UpdateMetadataEndpoint()
- .setHost("host1")
- .setPort(1223)
- .setSecurityProtocol(plaintext.id)
- .setListener(ListenerName.forSecurityProtocol(plaintext).value()));
-
- List<UpdateMetadataEndpoint> endpoints2 = new ArrayList<>();
- endpoints2.add(new UpdateMetadataEndpoint()
- .setHost("host1")
- .setPort(1244)
- .setSecurityProtocol(plaintext.id)
- .setListener(ListenerName.forSecurityProtocol(plaintext).value()));
- if (version > 0) {
- SecurityProtocol ssl = SecurityProtocol.SSL;
- endpoints2.add(new UpdateMetadataEndpoint()
- .setHost("host2")
- .setPort(1234)
- .setSecurityProtocol(ssl.id)
- .setListener(ListenerName.forSecurityProtocol(ssl).value()));
- endpoints2.add(new UpdateMetadataEndpoint()
- .setHost("host2")
- .setPort(1334)
- .setSecurityProtocol(ssl.id));
- if (version >= 3)
- endpoints2.get(1).setListener("CLIENT");
- }
-
- List<UpdateMetadataBroker> liveBrokers = asList(
- new UpdateMetadataBroker()
- .setId(0)
- .setEndpoints(endpoints1)
- .setRack(rack),
- new UpdateMetadataBroker()
- .setId(1)
- .setEndpoints(endpoints2)
- .setRack(rack)
- );
- return new UpdateMetadataRequest.Builder(version, 1, 10, 0, partitionStates,
- liveBrokers, topicIds).build();
- }
-
- private UpdateMetadataResponse createUpdateMetadataResponse() {
- return new UpdateMetadataResponse(new UpdateMetadataResponseData().setErrorCode(Errors.NONE.code()));
- }
-
private SaslHandshakeRequest createSaslHandshakeRequest(short version) {
return new SaslHandshakeRequest.Builder(
new SaslHandshakeRequestData().setMechanism("PLAIN")).build(version);
diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala
index 78ba1a02d47..f41684bbe9d 100755
--- a/core/src/main/scala/kafka/cluster/Partition.scala
+++ b/core/src/main/scala/kafka/cluster/Partition.scala
@@ -31,7 +31,6 @@ import org.apache.kafka.common.{DirectoryId, IsolationLevel, TopicIdPartition, T
import org.apache.kafka.common.errors._
import org.apache.kafka.common.message.AlterPartitionRequestData.BrokerState
import org.apache.kafka.common.message.{DescribeProducersResponseData, FetchResponseData}
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset
import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
@@ -327,7 +326,7 @@ class Partition(val topicPartition: TopicPartition,
// lock to prevent the follower replica log update while checking if the log dir could be replaced with future log.
private val futureLogLock = new Object()
- // The current epoch for the partition for KRaft controllers. The current ZK version for the legacy controllers.
+ // The current epoch for the partition for KRaft controllers.
@volatile private var partitionEpoch: Int = LeaderAndIsr.INITIAL_PARTITION_EPOCH
@volatile private var leaderEpoch: Int = LeaderAndIsr.INITIAL_LEADER_EPOCH - 1
// start offset for 'leaderEpoch' above (leader epoch of the current leader for this partition),
@@ -726,7 +725,7 @@ class Partition(val topicPartition: TopicPartition,
* from the time when this broker was the leader last time) and setting the new leader and ISR.
* If the leader replica id does not change, return false to indicate the replica manager.
*/
- def makeLeader(partitionState: LeaderAndIsrPartitionState,
+ def makeLeader(partitionState: LeaderAndIsrRequest.PartitionState,
highWatermarkCheckpoints: OffsetCheckpoints,
topicId: Option[Uuid],
targetDirectoryId: Option[Uuid] = None): Boolean = {
@@ -832,7 +831,7 @@ class Partition(val topicPartition: TopicPartition,
* replica manager that state is already correct and the become-follower steps can
* be skipped.
*/
- def makeFollower(partitionState: LeaderAndIsrPartitionState,
+ def makeFollower(partitionState: LeaderAndIsrRequest.PartitionState,
highWatermarkCheckpoints: OffsetCheckpoints,
topicId: Option[Uuid],
targetLogDirectoryId: Option[Uuid] = None): Boolean = {
@@ -880,7 +879,7 @@ class Partition(val topicPartition: TopicPartition,
}
}
- private def createLogInAssignedDirectoryId(partitionState: LeaderAndIsrPartitionState, highWatermarkCheckpoints: OffsetCheckpoints, topicId: Option[Uuid], targetLogDirectoryId: Option[Uuid]): Unit = {
+ private def createLogInAssignedDirectoryId(partitionState: LeaderAndIsrRequest.PartitionState, highWatermarkCheckpoints: OffsetCheckpoints, topicId: Option[Uuid], targetLogDirectoryId: Option[Uuid]): Unit = {
targetLogDirectoryId match {
case Some(directoryId) =>
if (logManager.onlineLogDirId(directoryId) || !logManager.hasOfflineLogDirs() || directoryId == DirectoryId.UNASSIGNED) {
diff --git a/core/src/main/scala/kafka/log/LogManager.scala b/core/src/main/scala/kafka/log/LogManager.scala
index 28bfa56adc8..fee4b95d2aa 100755
--- a/core/src/main/scala/kafka/log/LogManager.scala
+++ b/core/src/main/scala/kafka/log/LogManager.scala
@@ -35,7 +35,6 @@ import scala.jdk.CollectionConverters._
import scala.collection._
import scala.collection.mutable.ArrayBuffer
import scala.util.{Failure, Success, Try}
-import org.apache.kafka.common.requests.{AbstractControlRequest, LeaderAndIsrRequest}
import org.apache.kafka.image.TopicsImage
import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesEnsemble, PropertiesUtils}
@@ -1619,48 +1618,4 @@ object LogManager {
true
}
}
-
- /**
- * Find logs which should not be on the current broker, according to the full LeaderAndIsrRequest.
- *
- * @param brokerId The ID of the current broker.
- * @param request The full LeaderAndIsrRequest, containing all partitions owned by the broker.
- * @param logs A collection of Log objects.
- *
- * @return The topic partitions which are no longer needed on this broker.
- */
- def findStrayReplicas(
- brokerId: Int,
- request: LeaderAndIsrRequest,
- logs: Iterable[UnifiedLog]
- ): Iterable[TopicPartition] = {
- if (request.requestType() != AbstractControlRequest.Type.FULL) {
- throw new RuntimeException("Cannot use incremental LeaderAndIsrRequest to find strays.")
- }
- val partitions = new util.HashMap[TopicPartition, Uuid]()
- request.data().topicStates().forEach(topicState => {
- topicState.partitionStates().forEach(partition => {
- partitions.put(new TopicPartition(topicState.topicName(), partition.partitionIndex()),
- topicState.topicId())
- })
- })
- logs.flatMap { log =>
- val topicId = log.topicId.getOrElse {
- throw new RuntimeException(s"The log dir $log does not have a topic ID, " +
- "which is not allowed when running in KRaft mode.")
- }
- Option(partitions.get(log.topicPartition)) match {
- case Some(id) =>
- if (id.equals(topicId)) {
- None
- } else {
- info(s"Found stray log dir $log: this partition now exists with topic ID $id not $topicId.")
- Some(log.topicPartition)
- }
- case None =>
- info(s"Found stray log dir $log: this partition does not exist in the new full LeaderAndIsrRequest.")
- Some(log.topicPartition)
- }
- }
- }
}
diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala
index d0a2c4e811d..de658a82da0 100644
--- a/core/src/main/scala/kafka/network/SocketServer.scala
+++ b/core/src/main/scala/kafka/network/SocketServer.scala
@@ -803,9 +803,9 @@ private[kafka] object Processor {
if (apiVersionManager.isApiEnabled(header.apiKey, header.apiVersion)) {
header
} else if (header.isApiVersionSupported()) {
- throw new InvalidRequestException(s"Received request api key ${header.apiKey} with version ${header.apiVersion} which is not enabled")
+ throw new InvalidRequestException(s"Received request for disabled api with key ${header.apiKey.id} (${header.apiKey().name}) and version ${header.apiVersion}")
} else {
- throw new UnsupportedVersionException(s"Received request api key ${header.apiKey} with version ${header.apiVersion} which is not supported")
+ throw new UnsupportedVersionException(s"Received request for api with key ${header.apiKey.id} (${header.apiKey().name}) and unsupported version ${header.apiVersion}")
}
}
}
diff --git a/core/src/main/scala/kafka/server/AddPartitionsToTxnManager.scala b/core/src/main/scala/kafka/server/AddPartitionsToTxnManager.scala
index eab17214c64..79941ac957f 100644
--- a/core/src/main/scala/kafka/server/AddPartitionsToTxnManager.scala
+++ b/core/src/main/scala/kafka/server/AddPartitionsToTxnManager.scala
@@ -182,7 +182,7 @@ class AddPartitionsToTxnManager(
}
private def getTransactionCoordinator(partition: Int): Option[Node] = {
- metadataCache.getPartitionInfo(Topic.TRANSACTION_STATE_TOPIC_NAME, partition)
+ metadataCache.getLeaderAndIsr(Topic.TRANSACTION_STATE_TOPIC_NAME, partition)
.filter(_.leader != MetadataResponse.NO_LEADER_ID)
.flatMap(metadata => metadataCache.getAliveBrokerNode(metadata.leader, interBrokerListenerName))
}
diff --git a/core/src/main/scala/kafka/server/DelayedElectLeader.scala b/core/src/main/scala/kafka/server/DelayedElectLeader.scala
index e7171cff985..890f81526eb 100644
--- a/core/src/main/scala/kafka/server/DelayedElectLeader.scala
+++ b/core/src/main/scala/kafka/server/DelayedElectLeader.scala
@@ -74,7 +74,7 @@ class DelayedElectLeader(
private def updateWaiting(): Unit = {
val metadataCache = replicaManager.metadataCache
val completedPartitions = waitingPartitions.collect {
- case (tp, leader) if metadataCache.getPartitionInfo(tp.topic, tp.partition).exists(_.leader == leader) => tp
+ case (tp, leader) if metadataCache.getLeaderAndIsr(tp.topic, tp.partition).exists(_.leader == leader) => tp
}
completedPartitions.foreach { tp =>
waitingPartitions -= tp
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index 3cb1cc8dfd8..3f13f18dbca 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -297,7 +297,7 @@ class KafkaApis(val requestChannel: RequestChannel,
val topicWithValidPartitions = new OffsetCommitRequestData.OffsetCommitRequestTopic().setName(topic.name)
topic.partitions.forEach { partition =>
- if (metadataCache.getPartitionInfo(topic.name, partition.partitionIndex).nonEmpty) {
+ if (metadataCache.getLeaderAndIsr(topic.name, partition.partitionIndex).nonEmpty) {
topicWithValidPartitions.partitions.add(partition)
} else {
responseBuilder.addPartition(topic.name, partition.partitionIndex, Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -363,7 +363,7 @@ class KafkaApis(val requestChannel: RequestChannel,
(x.leaderReplicaIdOpt.getOrElse(-1), x.getLeaderEpoch)
case Left(x) =>
debug(s"Unable to retrieve local leaderId and Epoch with error $x, falling back to metadata cache")
- metadataCache.getPartitionInfo(tp.topic, tp.partition) match {
+ metadataCache.getLeaderAndIsr(tp.topic, tp.partition) match {
case Some(pinfo) => (pinfo.leader(), pinfo.leaderEpoch())
case None => (-1, -1)
}
@@ -2008,7 +2008,7 @@ class KafkaApis(val requestChannel: RequestChannel,
val topicWithValidPartitions = new TxnOffsetCommitRequestData.TxnOffsetCommitRequestTopic().setName(topic.name)
topic.partitions.forEach { partition =>
- if (metadataCache.getPartitionInfo(topic.name, partition.partitionIndex).nonEmpty) {
+ if (metadataCache.getLeaderAndIsr(topic.name, partition.partitionIndex).nonEmpty) {
topicWithValidPartitions.partitions.add(partition)
} else {
responseBuilder.addPartition(topic.name, partition.partitionIndex, Errors.UNKNOWN_TOPIC_OR_PARTITION)
@@ -2329,7 +2329,7 @@ class KafkaApis(val requestChannel: RequestChannel,
val topicWithValidPartitions = new OffsetDeleteRequestData.OffsetDeleteRequestTopic().setName(topic.name)
topic.partitions.forEach { partition =>
- if (metadataCache.getPartitionInfo(topic.name, partition.partitionIndex).nonEmpty) {
+ if (metadataCache.getLeaderAndIsr(topic.name, partition.partitionIndex).nonEmpty) {
topicWithValidPartitions.partitions.add(partition)
} else {
responseBuilder.addPartition(topic.name, partition.partitionIndex, Errors.UNKNOWN_TOPIC_OR_PARTITION)
diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala
index 4363e211a65..e29b8945932 100755
--- a/core/src/main/scala/kafka/server/MetadataCache.scala
+++ b/core/src/main/scala/kafka/server/MetadataCache.scala
@@ -19,9 +19,10 @@ package kafka.server
import kafka.server.metadata.KRaftMetadataCache
import org.apache.kafka.admin.BrokerMetadata
-import org.apache.kafka.common.message.{MetadataResponseData, UpdateMetadataRequestData}
+import org.apache.kafka.common.message.MetadataResponseData
import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.{Cluster, Node, TopicPartition, Uuid}
+import org.apache.kafka.metadata.LeaderAndIsr
import org.apache.kafka.server.common.{FinalizedFeatures, KRaftVersion, MetadataVersion}
import java.util
@@ -76,7 +77,7 @@ trait MetadataCache {
def getBrokerNodes(listenerName: ListenerName): Iterable[Node]
- def getPartitionInfo(topic: String, partitionId: Int): Option[UpdateMetadataRequestData.UpdateMetadataPartitionState]
+ def getLeaderAndIsr(topic: String, partitionId: Int): Option[LeaderAndIsr]
/**
* Return the number of partitions in the given topic, or None if the given topic does not exist.
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index 9504731f683..4f6895d1d5a 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -23,21 +23,18 @@ import kafka.log.remote.RemoteLogManager
import kafka.log.{LogManager, UnifiedLog}
import kafka.server.HostedPartition.Online
import kafka.server.QuotaFactory.QuotaManagers
-import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult, isListOffsetsTimestampUnsupported}
+import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, StopReplicaPartitionState, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult, isListOffsetsTimestampU [...]
import kafka.server.share.DelayedShareFetch
import kafka.utils._
import org.apache.kafka.common.errors._
import org.apache.kafka.common.internals.Topic
import org.apache.kafka.common.message.DeleteRecordsResponseData.DeleteRecordsPartitionResult
import org.apache.kafka.common.message.DescribeLogDirsResponseData.DescribeLogDirsTopic
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
-import org.apache.kafka.common.message.LeaderAndIsrResponseData.{LeaderAndIsrPartitionError, LeaderAndIsrTopicError}
import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartition, ListOffsetsTopic}
import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsPartitionResponse, ListOffsetsTopicResponse}
import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult}
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionState
-import org.apache.kafka.common.message.{DescribeLogDirsResponseData, DescribeProducersResponseData, FetchResponseData, LeaderAndIsrResponseData}
+import org.apache.kafka.common.message.{DescribeLogDirsResponseData, DescribeProducersResponseData, FetchResponseData}
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.protocol.Errors
@@ -261,6 +258,8 @@ object ReplicaManager {
timestamp < 0 &&
(!timestampMinSupportedVersion.contains(timestamp) || version < timestampMinSupportedVersion(timestamp))
}
+
+ case class StopReplicaPartitionState(leaderEpoch: Int, deletePartition: Boolean)
}
class ReplicaManager(val config: KafkaConfig,
@@ -419,42 +418,6 @@ class ReplicaManager(val config: KafkaConfig,
brokerTopicStats.removeMetrics(topic)
}
- private[server] def updateStrayLogs(strayPartitions: Iterable[TopicPartition]): Unit = {
- if (strayPartitions.isEmpty) {
- return
- }
- warn(s"Found stray partitions ${strayPartitions.mkString(",")}")
-
- // First, stop the partitions. This will shutdown the fetchers and other managers
- val partitionsToStop = strayPartitions.map(tp => new StopPartition(tp, false, false, false)).toSet
- stopPartitions(partitionsToStop).foreachEntry { (topicPartition, exception) =>
- error(s"Unable to stop stray partition $topicPartition", exception)
- }
-
- // Next, delete the in-memory partition state. Normally, stopPartitions would do this, but since we're not
- // actually deleting the log, so we can't rely on the "deleteLocalLog" behavior in stopPartitions.
- strayPartitions.foreach { topicPartition =>
- getPartition(topicPartition) match {
- case hostedPartition: HostedPartition.Online =>
- if (allPartitions.remove(topicPartition, hostedPartition)) {
- maybeRemoveTopicMetrics(topicPartition.topic)
- hostedPartition.partition.delete()
- }
- case _ =>
- }
- }
-
- // Mark the log as stray in-memory and rename the directory
- strayPartitions.foreach { tp =>
- logManager.getLog(tp).foreach(logManager.addStrayLog(tp, _))
- logManager.getLog(tp, isFuture = true).foreach(logManager.addStrayLog(tp, _))
- }
- logManager.asyncDelete(strayPartitions, isStray = true, (topicPartition, e) => {
- error(s"Failed to delete stray partition $topicPartition due to " +
- s"${e.getClass.getName} exception: ${e.getMessage}")
- })
- }
-
private def completeDelayedOperationsWhenNotPartitionLeader(topicPartition: TopicPartition, topicId: Option[Uuid]): Unit = {
val topicPartitionOperationKey = new TopicPartitionOperationKey(topicPartition)
delayedProducePurgatory.checkAndComplete(topicPartitionOperationKey)
@@ -545,7 +508,7 @@ class ReplicaManager(val config: KafkaConfig,
val stoppedPartitions = mutable.Buffer.empty[StopPartition]
partitionStates.foreachEntry { (topicPartition, partitionState) =>
- val deletePartition = partitionState.deletePartition()
+ val deletePartition = partitionState.deletePartition
getPartition(topicPartition) match {
case HostedPartition.Offline(_) =>
@@ -2064,14 +2027,14 @@ class ReplicaManager(val config: KafkaConfig,
stateChangeLogger.warn(s"Ignoring LeaderAndIsr request from controller $controllerId with " +
s"correlation id $correlationId since its controller epoch ${leaderAndIsrRequest.controllerEpoch} is old. " +
s"Latest known controller epoch is $controllerEpoch")
- leaderAndIsrRequest.getErrorResponse(0, Errors.STALE_CONTROLLER_EPOCH.exception)
+ leaderAndIsrRequest.getErrorResponse(Errors.STALE_CONTROLLER_EPOCH.exception)
} else {
val responseMap = new mutable.HashMap[TopicPartition, Errors]
controllerEpoch = leaderAndIsrRequest.controllerEpoch
val partitions = new mutable.HashSet[Partition]()
- val partitionsToBeLeader = new mutable.HashMap[Partition, LeaderAndIsrPartitionState]()
- val partitionsToBeFollower = new mutable.HashMap[Partition, LeaderAndIsrPartitionState]()
+ val partitionsToBeLeader = new mutable.HashMap[Partition, LeaderAndIsrRequest.PartitionState]()
+ val partitionsToBeFollower = new mutable.HashMap[Partition, LeaderAndIsrRequest.PartitionState]()
val topicIdUpdateFollowerPartitions = new mutable.HashSet[Partition]()
val allTopicPartitionsInRequest = new mutable.HashSet[TopicPartition]()
@@ -2109,7 +2072,7 @@ class ReplicaManager(val config: KafkaConfig,
s" match the topic ID for partition $topicPartition received: " +
s"${requestTopicId.get}.")
responseMap.put(topicPartition, Errors.INCONSISTENT_TOPIC_ID)
- } else if (requestLeaderEpoch > currentLeaderEpoch || (requestLeaderEpoch == currentLeaderEpoch && leaderAndIsrRequest.isKRaftController)) {
+ } else if (requestLeaderEpoch >= currentLeaderEpoch) {
// If the leader epoch is valid record the epoch of the controller that made the leadership decision.
// This is useful while updating the isr to maintain the decision maker controller's epoch in the zookeeper path
if (partitionState.replicas.contains(localBrokerId)) {
@@ -2196,28 +2159,17 @@ class ReplicaManager(val config: KafkaConfig,
onLeadershipChange(partitionsBecomeLeader, partitionsBecomeFollower)
- val data = new LeaderAndIsrResponseData().setErrorCode(Errors.NONE.code)
- if (leaderAndIsrRequest.version < 5) {
- responseMap.foreachEntry { (tp, error) =>
- data.partitionErrors.add(new LeaderAndIsrPartitionError()
- .setTopicName(tp.topic)
- .setPartitionIndex(tp.partition)
- .setErrorCode(error.code))
- }
- } else {
- responseMap.foreachEntry { (tp, error) =>
- val topicId = topicIds.get(tp.topic)
- var topic = data.topics.find(topicId)
- if (topic == null) {
- topic = new LeaderAndIsrTopicError().setTopicId(topicId)
- data.topics.add(topic)
- }
- topic.partitionErrors.add(new LeaderAndIsrPartitionError()
- .setPartitionIndex(tp.partition)
- .setErrorCode(error.code))
+ val topics = new util.LinkedHashMap[Uuid, util.List[LeaderAndIsrResponse.PartitionError]]
+ responseMap.foreachEntry { (tp, error) =>
+ val topicId = topicIds.get(tp.topic)
+ var partitionErrors = topics.get(topicId)
+ if (partitionErrors == null) {
+ partitionErrors = new util.ArrayList[LeaderAndIsrResponse.PartitionError]()
+ topics.put(topicId, partitionErrors)
}
+ partitionErrors.add(new LeaderAndIsrResponse.PartitionError(tp.partition(), error.code))
}
- new LeaderAndIsrResponse(data, leaderAndIsrRequest.version)
+ new LeaderAndIsrResponse(Errors.NONE, topics)
}
}
val endMs = time.milliseconds()
@@ -2307,7 +2259,7 @@ class ReplicaManager(val config: KafkaConfig,
*/
private def makeLeaders(controllerId: Int,
controllerEpoch: Int,
- partitionStates: Map[Partition, LeaderAndIsrPartitionState],
+ partitionStates: Map[Partition, LeaderAndIsrRequest.PartitionState],
correlationId: Int,
responseMap: mutable.Map[TopicPartition, Errors],
highWatermarkCheckpoints: OffsetCheckpoints,
@@ -2389,7 +2341,7 @@ class ReplicaManager(val config: KafkaConfig,
*/
private def makeFollowers(controllerId: Int,
controllerEpoch: Int,
- partitionStates: Map[Partition, LeaderAndIsrPartitionState],
+ partitionStates: Map[Partition, LeaderAndIsrRequest.PartitionState],
correlationId: Int,
responseMap: mutable.Map[TopicPartition, Errors],
highWatermarkCheckpoints: OffsetCheckpoints,
diff --git a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala
index 5d7484714d0..614077e2ca4 100644
--- a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala
+++ b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala
@@ -26,13 +26,12 @@ import org.apache.kafka.common.errors.InvalidTopicException
import org.apache.kafka.common.internals.Topic
import org.apache.kafka.common.message.DescribeTopicPartitionsResponseData.{Cursor, DescribeTopicPartitionsResponsePartition, DescribeTopicPartitionsResponseTopic}
import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic}
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState
import org.apache.kafka.common.message._
import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.requests.MetadataResponse
import org.apache.kafka.image.MetadataImage
-import org.apache.kafka.metadata.{BrokerRegistration, PartitionRegistration, Replicas}
+import org.apache.kafka.metadata.{BrokerRegistration, LeaderAndIsr, PartitionRegistration, Replicas}
import org.apache.kafka.server.common.{FinalizedFeatures, KRaftVersion, MetadataVersion}
import java.util
@@ -385,19 +384,11 @@ class KRaftMetadataCache(
_currentImage.cluster().brokers().values().asScala.flatMap(_.node(listenerName.value()).asScala).toSeq
}
- // Does NOT include offline replica metadata
- override def getPartitionInfo(topicName: String, partitionId: Int): Option[UpdateMetadataPartitionState] = {
+ override def getLeaderAndIsr(topicName: String, partitionId: Int): Option[LeaderAndIsr] = {
Option(_currentImage.topics().getTopic(topicName)).
flatMap(topic => Option(topic.partitions().get(partitionId))).
- flatMap(partition => Some(new UpdateMetadataPartitionState().
- setTopicName(topicName).
- setPartitionIndex(partitionId).
- setControllerEpoch(-1). // Controller epoch is not stored in the cache.
- setLeader(partition.leader).
- setLeaderEpoch(partition.leaderEpoch).
- setIsr(Replicas.toList(partition.isr)).
- setZkVersion(partition.partitionEpoch).
- setReplicas(Replicas.toList(partition.replicas))))
+ flatMap(partition => Some(new LeaderAndIsr(partition.leader, partition.leaderEpoch,
+ util.Arrays.asList(partition.isr.map(i => i: java.lang.Integer): _*), partition.leaderRecoveryState, partition.partitionEpoch)))
}
override def numPartitions(topicName: String): Option[Int] = {
diff --git a/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java b/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java
index e48e830175b..55bc36cba41 100644
--- a/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java
+++ b/core/src/test/java/kafka/server/handlers/DescribeTopicPartitionsRequestHandlerTest.java
@@ -31,8 +31,6 @@ import org.apache.kafka.common.memory.MemoryPool;
import org.apache.kafka.common.message.DescribeTopicPartitionsRequestData;
import org.apache.kafka.common.message.DescribeTopicPartitionsResponseData;
import org.apache.kafka.common.message.DescribeTopicPartitionsResponseData.DescribeTopicPartitionsResponseTopic;
-import org.apache.kafka.common.message.UpdateMetadataRequestData;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker;
import org.apache.kafka.common.metadata.PartitionRecord;
import org.apache.kafka.common.metadata.RegisterBrokerRecord;
import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint;
@@ -106,16 +104,13 @@ class DescribeTopicPartitionsRequestHandlerTest {
};
ListenerName plaintextListener = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT);
- UpdateMetadataBroker broker = new UpdateMetadataBroker()
- .setId(0)
- .setRack("rack")
- .setEndpoints(Arrays.asList(
- new UpdateMetadataRequestData.UpdateMetadataEndpoint()
- .setHost("broker0")
- .setPort(9092)
- .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
- .setListener(plaintextListener.value())
- ));
+ String rack = "rack";
+ int brokerId = 0;
+ BrokerEndpoint brokerEndpoint = new BrokerEndpoint()
+ .setName(plaintextListener.value())
+ .setHost("broker0")
+ .setPort(9092)
+ .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id);
@Test
void testDescribeTopicPartitionsRequest() {
@@ -150,19 +145,14 @@ class DescribeTopicPartitionsRequestHandlerTest {
topicIds.put(unauthorizedTopic, unauthorizedTopicId);
BrokerEndpointCollection collection = new BrokerEndpointCollection();
- collection.add(new BrokerEndpoint()
- .setName(broker.endpoints().get(0).listener())
- .setHost(broker.endpoints().get(0).host())
- .setPort(broker.endpoints().get(0).port())
- .setSecurityProtocol(broker.endpoints().get(0).securityProtocol())
- );
+ collection.add(brokerEndpoint);
List<ApiMessage> records = Arrays.asList(
new RegisterBrokerRecord()
- .setBrokerId(broker.id())
+ .setBrokerId(brokerId)
.setBrokerEpoch(0)
.setIncarnationId(Uuid.randomUuid())
.setEndPoints(collection)
- .setRack(broker.rack())
+ .setRack(rack)
.setFenced(false),
new TopicRecord().setName(authorizedTopic).setTopicId(topicIds.get(authorizedTopic)),
new TopicRecord().setName(unauthorizedTopic).setTopicId(topicIds.get(unauthorizedTopic)),
@@ -352,19 +342,14 @@ class DescribeTopicPartitionsRequestHandlerTest {
topicIds.put(authorizedTopic2, authorizedTopicId2);
BrokerEndpointCollection collection = new BrokerEndpointCollection();
- collection.add(new BrokerEndpoint()
- .setName(broker.endpoints().get(0).listener())
- .setHost(broker.endpoints().get(0).host())
- .setPort(broker.endpoints().get(0).port())
- .setSecurityProtocol(broker.endpoints().get(0).securityProtocol())
- );
+ collection.add(brokerEndpoint);
List<ApiMessage> records = Arrays.asList(
new RegisterBrokerRecord()
- .setBrokerId(broker.id())
+ .setBrokerId(brokerId)
.setBrokerEpoch(0)
.setIncarnationId(Uuid.randomUuid())
.setEndPoints(collection)
- .setRack(broker.rack())
+ .setRack(rack)
.setFenced(false),
new TopicRecord().setName(authorizedTopic).setTopicId(topicIds.get(authorizedTopic)),
new TopicRecord().setName(authorizedTopic2).setTopicId(topicIds.get(authorizedTopic2)),
@@ -563,4 +548,4 @@ class DescribeTopicPartitionsRequestHandlerTest {
TestUtils.setIbpVersion(properties, MetadataVersion.latestProduction());
return new KafkaConfig(properties);
}
-}
\ No newline at end of file
+}
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 3ee420f2c49..5f23c54ee88 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -116,7 +116,6 @@ class AuthorizerIntegrationTest extends AbstractAuthorizerIntegrationTest {
ApiKeys.FIND_COORDINATOR -> ((resp: FindCoordinatorResponse) => {
Errors.forCode(resp.data.coordinators.asScala.find(g => group == g.key).head.errorCode)
}),
- ApiKeys.UPDATE_METADATA -> ((resp: requests.UpdateMetadataResponse) => resp.error),
ApiKeys.JOIN_GROUP -> ((resp: JoinGroupResponse) => resp.error),
ApiKeys.SYNC_GROUP -> ((resp: SyncGroupResponse) => Errors.forCode(resp.data.errorCode)),
ApiKeys.DESCRIBE_GROUPS -> ((resp: DescribeGroupsResponse) => {
@@ -125,12 +124,6 @@ class AuthorizerIntegrationTest extends AbstractAuthorizerIntegrationTest {
ApiKeys.HEARTBEAT -> ((resp: HeartbeatResponse) => resp.error),
ApiKeys.LEAVE_GROUP -> ((resp: LeaveGroupResponse) => resp.error),
ApiKeys.DELETE_GROUPS -> ((resp: DeleteGroupsResponse) => resp.get(group)),
- ApiKeys.LEADER_AND_ISR -> ((resp: requests.LeaderAndIsrResponse) => Errors.forCode(
- resp.topics.asScala.find(t => topicNames(t.topicId) == tp.topic).get.partitionErrors.asScala.find(
- p => p.partitionIndex == tp.partition).get.errorCode)),
- ApiKeys.STOP_REPLICA -> ((resp: requests.StopReplicaResponse) => Errors.forCode(
- resp.partitionErrors.asScala.find(pe => pe.topicName == tp.topic && pe.partitionIndex == tp.partition).get.errorCode)),
- ApiKeys.CONTROLLED_SHUTDOWN -> ((resp: requests.ControlledShutdownResponse) => resp.error),
ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(topic).errorCode)),
ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(topic).errorCode)),
ApiKeys.DELETE_RECORDS -> ((resp: requests.DeleteRecordsResponse) => Errors.forCode(
@@ -1241,7 +1234,7 @@ class AuthorizerIntegrationTest extends AbstractAuthorizerIntegrationTest {
waitUntilTrue(() => {
consumer.poll(Duration.ofMillis(50L))
brokers.forall { broker =>
- broker.metadataCache.getPartitionInfo(newTopic, 0) match {
+ broker.metadataCache.getLeaderAndIsr(newTopic, 0) match {
case Some(partitionState) => FetchRequest.isValidBrokerId(partitionState.leader)
case _ => false
}
@@ -1254,7 +1247,7 @@ class AuthorizerIntegrationTest extends AbstractAuthorizerIntegrationTest {
def testCreatePermissionMetadataRequestAutoCreate(quorum: String): Unit = {
val readAcls = topicReadAcl(topicResource)
addAndVerifyAcls(readAcls, topicResource)
- brokers.foreach(b => assertEquals(None, b.metadataCache.getPartitionInfo(topic, 0)))
+ brokers.foreach(b => assertEquals(None, b.metadataCache.getLeaderAndIsr(topic, 0)))
val metadataRequest = new MetadataRequest.Builder(List(topic).asJava, true).build()
val metadataResponse = connectAndReceive[MetadataResponse](metadataRequest)
diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
index 600833ca4ae..fe5a4047d63 100644
--- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
+++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
@@ -1355,7 +1355,7 @@ class KRaftClusterTest {
// Shut down broker0 and wait until the ISR of foo-0 is set to [1, 2]
broker0.shutdown()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(1, 2), info.get.isr().asScala.toSet)
}
@@ -1369,7 +1369,7 @@ class KRaftClusterTest {
// Start up broker0 and wait until the ISR of foo-0 is set to [0, 1, 2]
broker0.startup()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(0, 1, 2), info.get.isr().asScala.toSet)
}
@@ -1410,7 +1410,7 @@ class KRaftClusterTest {
// Shut down broker0 and wait until the ISR of foo-0 is set to [1, 2]
broker0.shutdown()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(1, 2), info.get.isr().asScala.toSet)
}
@@ -1424,7 +1424,7 @@ class KRaftClusterTest {
// Start up broker0 and wait until the ISR of foo-0 is set to [0, 1, 2]
broker0.startup()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(0, 1, 2), info.get.isr().asScala.toSet)
assertTrue(broker0.logManager.getLog(foo0, isFuture = true).isEmpty)
@@ -1475,7 +1475,7 @@ class KRaftClusterTest {
// Shut down broker0 and wait until the ISR of foo-0 is set to [1, 2]
broker0.shutdown()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(1, 2), info.get.isr().asScala.toSet)
}
@@ -1501,7 +1501,7 @@ class KRaftClusterTest {
// Start up broker0 and wait until the ISR of foo-0 is set to [0, 1, 2]
broker0.startup()
TestUtils.retry(60000) {
- val info = broker1.metadataCache.getPartitionInfo("foo", 0)
+ val info = broker1.metadataCache.getLeaderAndIsr("foo", 0)
assertTrue(info.isDefined)
assertEquals(Set(0, 1, 2), info.get.isr().asScala.toSet)
assertTrue(broker0.logManager.getLog(foo0, isFuture = true).isEmpty)
diff --git a/core/src/test/scala/kafka/server/LocalLeaderEndPointTest.scala b/core/src/test/scala/kafka/server/LocalLeaderEndPointTest.scala
index 25e32ca4d79..86ffdfb681c 100644
--- a/core/src/test/scala/kafka/server/LocalLeaderEndPointTest.scala
+++ b/core/src/test/scala/kafka/server/LocalLeaderEndPointTest.scala
@@ -21,11 +21,10 @@ import kafka.server.QuotaFactory.QuotaManagers
import kafka.utils.{CoreUtils, Logging, TestUtils}
import org.apache.kafka.common.compress.Compression
import org.apache.kafka.common.{Node, TopicPartition, Uuid}
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset
import org.apache.kafka.common.metrics.Metrics
-import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.record.{MemoryRecords, SimpleRecord}
import org.apache.kafka.common.requests.LeaderAndIsrRequest
import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
@@ -223,8 +222,8 @@ class LocalLeaderEndPointTest extends Logging {
private def buildLeaderAndIsrRequest(leaderEpoch: Int): LeaderAndIsrRequest = {
val brokerList = Seq[Integer](sourceBroker.id).asJava
val topicIds = Collections.singletonMap(topic, topicId)
- new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, 0,
- Seq(new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.Builder(0, 0, 0,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(topicPartition.partition())
.setControllerEpoch(0)
diff --git a/core/src/test/scala/unit/kafka/cluster/AbstractPartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/AbstractPartitionTest.scala
index 3c8e20701b0..2cd271e1fe4 100644
--- a/core/src/test/scala/unit/kafka/cluster/AbstractPartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/AbstractPartitionTest.scala
@@ -23,7 +23,7 @@ import kafka.utils.TestUtils
import kafka.utils.TestUtils.MockAlterPartitionManager
import org.apache.kafka.common.{TopicPartition, Uuid}
import org.apache.kafka.common.config.TopicConfig
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.requests.LeaderAndIsrRequest
import org.apache.kafka.common.utils.Utils
import org.apache.kafka.server.common.MetadataVersion
import org.apache.kafka.server.config.ReplicationConfigs
@@ -126,7 +126,7 @@ class AbstractPartitionTest {
val isr = replicas
if (isLeader) {
- assertTrue(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -136,7 +136,7 @@ class AbstractPartitionTest {
.setIsNew(true), offsetCheckpoints, None), "Expected become leader transition to succeed")
assertEquals(leaderEpoch, partition.getLeaderEpoch)
} else {
- assertTrue(partition.makeFollower(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeFollower(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(remoteReplicaId)
.setLeaderEpoch(leaderEpoch)
diff --git a/core/src/test/scala/unit/kafka/cluster/AssignmentStateTest.scala b/core/src/test/scala/unit/kafka/cluster/AssignmentStateTest.scala
index ce8c567353d..c34a7ac7536 100644
--- a/core/src/test/scala/unit/kafka/cluster/AssignmentStateTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/AssignmentStateTest.scala
@@ -16,7 +16,7 @@
*/
package kafka.cluster
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.requests.LeaderAndIsrRequest
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{Arguments, MethodSource}
@@ -89,7 +89,7 @@ class AssignmentStateTest extends AbstractPartitionTest {
original: Seq[Int], isUnderReplicated: Boolean): Unit = {
val controllerEpoch = 3
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(6)
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala
index 8b874424541..d7fa615ba75 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionLockTest.scala
@@ -27,10 +27,9 @@ import kafka.server.metadata.MockConfigRepository
import kafka.utils._
import org.apache.kafka.common.TopicIdPartition
import org.apache.kafka.common.config.TopicConfig
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
import org.apache.kafka.common.protocol.ApiKeys
import org.apache.kafka.common.record.{MemoryRecords, SimpleRecord}
-import org.apache.kafka.common.requests.FetchRequest
+import org.apache.kafka.common.requests.{FetchRequest, LeaderAndIsrRequest}
import org.apache.kafka.common.utils.Utils
import org.apache.kafka.common.{TopicPartition, Uuid}
import org.apache.kafka.coordinator.transaction.TransactionLogConfig
@@ -144,7 +143,7 @@ class PartitionLockTest extends Logging {
val replicaToCheck = 3
val firstReplicaSet = Seq[Integer](3, 4, 5).asJava
val secondReplicaSet = Seq[Integer](1, 2, 3).asJava
- def partitionState(replicas: java.util.List[Integer]) = new LeaderAndIsrPartitionState()
+ def partitionState(replicas: java.util.List[Integer]) = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(1)
.setLeader(replicas.get(0))
.setLeaderEpoch(1)
@@ -350,7 +349,7 @@ class PartitionLockTest extends Logging {
val replicas = (0 to numReplicaFetchers).map(i => Integer.valueOf(brokerId + i)).toList.asJava
val isr = replicas
- assertTrue(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
diff --git a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
index d05b3ee14a1..81bdb42db48 100644
--- a/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
+++ b/core/src/test/scala/unit/kafka/cluster/PartitionTest.scala
@@ -23,11 +23,10 @@ import kafka.server._
import kafka.utils._
import org.apache.kafka.common.errors.{ApiException, FencedLeaderEpochException, InconsistentTopicIdException, InvalidTxnStateException, NotLeaderOrFollowerException, OffsetNotAvailableException, OffsetOutOfRangeException, UnknownLeaderEpochException}
import org.apache.kafka.common.message.{AlterPartitionResponseData, FetchResponseData}
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
import org.apache.kafka.common.protocol.{ApiKeys, Errors}
import org.apache.kafka.common.record.FileRecords.TimestampAndOffset
import org.apache.kafka.common.record._
-import org.apache.kafka.common.requests.{AlterPartitionResponse, FetchRequest, ListOffsetsRequest, RequestHeader}
+import org.apache.kafka.common.requests.{AlterPartitionResponse, FetchRequest, LeaderAndIsrRequest, ListOffsetsRequest, RequestHeader}
import org.apache.kafka.common.utils.Time
import org.apache.kafka.common.{DirectoryId, IsolationLevel, TopicPartition, Uuid}
import org.apache.kafka.server.config.ReplicationConfigs
@@ -313,7 +312,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val partitionEpoch = 1
- assertTrue(partition.makeFollower(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeFollower(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -356,7 +355,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val partitionEpoch = 1
- assertTrue(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch)
@@ -392,7 +391,7 @@ class PartitionTest extends AbstractPartitionTest {
val newPartitionEpoch = partitionEpoch + 1
val addingReplicas = List(addingReplica1, addingReplica2)
- assertFalse(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertFalse(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch)
@@ -491,7 +490,7 @@ class PartitionTest extends AbstractPartitionTest {
appendThread.start()
TestUtils.waitUntilTrue(() => appendSemaphore.hasQueuedThreads, "follower log append is not called.")
- val partitionState = new LeaderAndIsrPartitionState()
+ val partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(2)
.setLeaderEpoch(1)
@@ -790,7 +789,7 @@ class PartitionTest extends AbstractPartitionTest {
new SimpleRecord(20,"k4".getBytes, "v2".getBytes),
new SimpleRecord(21,"k5".getBytes, "v3".getBytes)))
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch)
@@ -861,7 +860,7 @@ class PartitionTest extends AbstractPartitionTest {
assertEquals(Right(None), fetchOffsetsForTimestamp(30, Some(IsolationLevel.READ_UNCOMMITTED)))
// Make into a follower
- val followerState = new LeaderAndIsrPartitionState()
+ val followerState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(follower2)
.setLeaderEpoch(leaderEpoch + 1)
@@ -873,7 +872,7 @@ class PartitionTest extends AbstractPartitionTest {
assertTrue(partition.makeFollower(followerState, offsetCheckpoints, None))
// Back to leader, this resets the startLogOffset for this epoch (to 2), we're now in the fault condition
- val newLeaderState = new LeaderAndIsrPartitionState()
+ val newLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch + 2)
@@ -1006,7 +1005,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
- assertTrue(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1084,7 +1083,7 @@ class PartitionTest extends AbstractPartitionTest {
@Test
def testMakeFollowerWithNoLeaderIdChange(): Unit = {
// Start off as follower
- var partitionState = new LeaderAndIsrPartitionState()
+ var partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(1)
.setLeaderEpoch(1)
@@ -1095,7 +1094,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.makeFollower(partitionState, offsetCheckpoints, None)
// Request with same leader and epoch increases by only 1, do become-follower steps
- partitionState = new LeaderAndIsrPartitionState()
+ partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(1)
.setLeaderEpoch(4)
@@ -1106,7 +1105,7 @@ class PartitionTest extends AbstractPartitionTest {
assertTrue(partition.makeFollower(partitionState, offsetCheckpoints, None))
// Request with same leader and same epoch, skip become-follower steps
- partitionState = new LeaderAndIsrPartitionState()
+ partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(1)
.setLeaderEpoch(4)
@@ -1133,7 +1132,7 @@ class PartitionTest extends AbstractPartitionTest {
val batch3 = TestUtils.records(records = List(new SimpleRecord("k6".getBytes, "v1".getBytes),
new SimpleRecord("k7".getBytes, "v2".getBytes)))
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch)
@@ -1160,7 +1159,7 @@ class PartitionTest extends AbstractPartitionTest {
assertEquals(lastOffsetOfFirstBatch + 1, partition.log.get.highWatermark, "Expected leader's HW")
// current leader becomes follower and then leader again (without any new records appended)
- val followerState = new LeaderAndIsrPartitionState()
+ val followerState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(follower2)
.setLeaderEpoch(leaderEpoch + 1)
@@ -1170,7 +1169,7 @@ class PartitionTest extends AbstractPartitionTest {
.setIsNew(false)
partition.makeFollower(followerState, offsetCheckpoints, None)
- val newLeaderState = new LeaderAndIsrPartitionState()
+ val newLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch + 2)
@@ -1252,7 +1251,7 @@ class PartitionTest extends AbstractPartitionTest {
assertFalse(partition.isAtMinIsr)
// Make isr set to only have leader to trigger AtMinIsr (default min isr config is 1)
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leader)
.setLeaderEpoch(leaderEpoch)
@@ -1279,7 +1278,7 @@ class PartitionTest extends AbstractPartitionTest {
alterPartitionManager)
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -1291,7 +1290,7 @@ class PartitionTest extends AbstractPartitionTest {
topicId = None)
assertFalse(partition.isUnderMinIsr)
- val LeaderState = new LeaderAndIsrPartitionState()
+ val LeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(1)
@@ -1319,7 +1318,7 @@ class PartitionTest extends AbstractPartitionTest {
val initializeTimeMs = time.milliseconds()
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1383,7 +1382,7 @@ class PartitionTest extends AbstractPartitionTest {
val initializeTimeMs = time.milliseconds()
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1417,7 +1416,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1468,7 +1467,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1531,7 +1530,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1603,7 +1602,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1645,7 +1644,7 @@ class PartitionTest extends AbstractPartitionTest {
// Controller shrinks the ISR after
assertFalse(
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1709,7 +1708,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1815,7 +1814,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1906,7 +1905,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -1971,7 +1970,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -2126,7 +2125,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -2208,7 +2207,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
assertTrue(partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -2686,7 +2685,7 @@ class PartitionTest extends AbstractPartitionTest {
val controllerEpoch = 3
val replicas = List[Integer](brokerId, brokerId + 1).asJava
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(6)
@@ -2704,7 +2703,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 5
val topicId = Uuid.randomUuid()
val replicas = List[Integer](brokerId, brokerId + 1).asJava
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -2748,7 +2747,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 5
val topicId = Uuid.randomUuid()
val replicas = List[Integer](brokerId, brokerId + 1).asJava
- val leaderState = new LeaderAndIsrPartitionState()
+ val leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -2825,7 +2824,7 @@ class PartitionTest extends AbstractPartitionTest {
val replicas = List[Integer](brokerId, brokerId + 1, brokerId + 2).asJava
val isr = List[Integer](brokerId, brokerId + 1).asJava
- var leaderState = new LeaderAndIsrPartitionState()
+ var leaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(6)
@@ -3028,7 +3027,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val topicId = Uuid.randomUuid()
- val initialLeaderState = new LeaderAndIsrPartitionState()
+ val initialLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3061,7 +3060,7 @@ class PartitionTest extends AbstractPartitionTest {
// makeLeader is called again with the same leader epoch but with
// a newer partition epoch. This can happen in KRaft when a partition
// is reassigned. The leader epoch is not bumped when we add replicas.
- val updatedLeaderState = new LeaderAndIsrPartitionState()
+ val updatedLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3092,7 +3091,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val topicId = Uuid.randomUuid()
- val initialLeaderState = new LeaderAndIsrPartitionState()
+ val initialLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3118,7 +3117,7 @@ class PartitionTest extends AbstractPartitionTest {
// makeLeader is called again with the same leader epoch but with
// a newer partition epoch.
- val updatedLeaderState = new LeaderAndIsrPartitionState()
+ val updatedLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3143,7 +3142,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val topicId = Uuid.randomUuid()
- val initialLeaderState = new LeaderAndIsrPartitionState()
+ val initialLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3158,7 +3157,7 @@ class PartitionTest extends AbstractPartitionTest {
// makeLeader is called again with the same leader epoch but with
// a older partition epoch.
- val updatedLeaderState = new LeaderAndIsrPartitionState()
+ val updatedLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(leaderId)
.setLeaderEpoch(leaderEpoch)
@@ -3181,7 +3180,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 8
val topicId = Uuid.randomUuid()
- val initialFollowerState = new LeaderAndIsrPartitionState()
+ val initialFollowerState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(followerId)
.setLeaderEpoch(leaderEpoch)
@@ -3196,7 +3195,7 @@ class PartitionTest extends AbstractPartitionTest {
// makeLeader is called again with the same leader epoch but with
// a older partition epoch.
- val updatedFollowerState = new LeaderAndIsrPartitionState()
+ val updatedFollowerState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(followerId)
.setLeaderEpoch(leaderEpoch)
@@ -3220,7 +3219,7 @@ class PartitionTest extends AbstractPartitionTest {
val topicId = Uuid.randomUuid()
// The local replica is the leader.
- val initialLeaderState = new LeaderAndIsrPartitionState()
+ val initialLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(localReplica)
.setLeaderEpoch(1)
@@ -3238,7 +3237,7 @@ class PartitionTest extends AbstractPartitionTest {
assertEquals(replicas, partition.assignmentState.replicas)
// The local replica becomes a follower.
- val updatedLeaderState = new LeaderAndIsrPartitionState()
+ val updatedLeaderState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(remoteReplica1)
.setLeaderEpoch(2)
@@ -3261,7 +3260,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3331,7 +3330,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3352,7 +3351,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3394,7 +3393,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3418,7 +3417,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, topicId = None)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3444,7 +3443,7 @@ class PartitionTest extends AbstractPartitionTest {
assertTrue(partition.log.isDefined)
partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(brokerId)
.setLeaderEpoch(0)
@@ -3514,7 +3513,7 @@ class PartitionTest extends AbstractPartitionTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
- assertTrue(partition.makeLeader(new LeaderAndIsrPartitionState()
+ assertTrue(partition.makeLeader(new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3579,7 +3578,7 @@ class PartitionTest extends AbstractPartitionTest {
topicId
)
val newLeader = partition.makeLeader(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3762,7 +3761,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3807,7 +3806,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3852,7 +3851,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3897,7 +3896,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3942,7 +3941,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
@@ -3988,7 +3987,7 @@ class PartitionTest extends AbstractPartitionTest {
val leaderEpoch = 1
val replicas = List[Integer](brokerId, brokerId + 1).asJava
val isr = replicas
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(controllerEpoch)
.setLeader(brokerId)
.setLeaderEpoch(leaderEpoch)
diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
index b4d3bee35d1..e08e5775c90 100755
--- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala
@@ -282,7 +282,7 @@ class UncleanLeaderElectionTest extends QuorumTestHarness {
produceMessage(brokers, topic, "third")
//make sure follower server joins the ISR
TestUtils.waitUntilTrue(() => {
- val partitionInfoOpt = followerServer.metadataCache.getPartitionInfo(topic, partitionId)
+ val partitionInfoOpt = followerServer.metadataCache.getLeaderAndIsr(topic, partitionId)
partitionInfoOpt.isDefined && partitionInfoOpt.get.isr.contains(followerId)
}, "Inconsistent metadata after first server startup")
@@ -424,9 +424,9 @@ class UncleanLeaderElectionTest extends QuorumTestHarness {
}
private def waitForNoLeaderAndIsrHasOldLeaderId(metadataCache: MetadataCache, leaderId: Int): Unit = {
- waitUntilTrue(() => metadataCache.getPartitionInfo(topic, partitionId).isDefined &&
- metadataCache.getPartitionInfo(topic, partitionId).get.leader() == LeaderConstants.NO_LEADER &&
- java.util.Arrays.asList(leaderId).equals(metadataCache.getPartitionInfo(topic, partitionId).get.isr()),
+ waitUntilTrue(() => metadataCache.getLeaderAndIsr(topic, partitionId).isDefined &&
+ metadataCache.getLeaderAndIsr(topic, partitionId).get.leader() == LeaderConstants.NO_LEADER &&
+ java.util.Arrays.asList(leaderId).equals(metadataCache.getLeaderAndIsr(topic, partitionId).get.isr()),
"Timed out waiting for broker metadata cache updates the info for topic partition:" + topicPartition)
}
}
diff --git a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
index e28d1425430..dcf3aaf3d0f 100755
--- a/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogManagerTest.scala
@@ -23,9 +23,6 @@ import kafka.utils._
import org.apache.directory.api.util.FileUtils
import org.apache.kafka.common.config.TopicConfig
import org.apache.kafka.common.errors.OffsetOutOfRangeException
-import org.apache.kafka.common.message.LeaderAndIsrRequestData
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrTopicState
-import org.apache.kafka.common.requests.{AbstractControlRequest, LeaderAndIsrRequest}
import org.apache.kafka.common.utils.{Time, Utils}
import org.apache.kafka.common.{DirectoryId, KafkaException, TopicIdPartition, TopicPartition, Uuid}
import org.apache.kafka.coordinator.transaction.TransactionLogConfig
@@ -1296,47 +1293,6 @@ class LogManagerTest {
assertTrue(LogManager.isStrayKraftReplica(0, topicsImage(Seq()), log))
}
- @Test
- def testFindStrayReplicasInEmptyLAIR(): Unit = {
- val onDisk = Seq(foo0, foo1, bar0, bar1, baz0, baz1, baz2, quux0)
- val expected = onDisk.map(_.topicPartition()).toSet
- assertEquals(expected,
- LogManager.findStrayReplicas(0,
- createLeaderAndIsrRequestForStrayDetection(Seq()),
- onDisk.map(mockLog)).toSet)
- }
-
- @Test
- def testFindNoStrayReplicasInFullLAIR(): Unit = {
- val onDisk = Seq(foo0, foo1, bar0, bar1, baz0, baz1, baz2, quux0)
- assertEquals(Set(),
- LogManager.findStrayReplicas(0,
- createLeaderAndIsrRequestForStrayDetection(onDisk),
- onDisk.map(mockLog)).toSet)
- }
-
- @Test
- def testFindSomeStrayReplicasInFullLAIR(): Unit = {
- val onDisk = Seq(foo0, foo1, bar0, bar1, baz0, baz1, baz2, quux0)
- val present = Seq(foo0, bar0, bar1, quux0)
- val expected = Seq(foo1, baz0, baz1, baz2).map(_.topicPartition()).toSet
- assertEquals(expected,
- LogManager.findStrayReplicas(0,
- createLeaderAndIsrRequestForStrayDetection(present),
- onDisk.map(mockLog)).toSet)
- }
-
- @Test
- def testTopicRecreationInFullLAIR(): Unit = {
- val onDisk = Seq(foo0, foo1, bar0, bar1, baz0, baz1, baz2, quux0)
- val present = Seq(recreatedFoo0, recreatedFoo1, bar0, baz0, baz1, baz2, quux0)
- val expected = Seq(foo0, foo1, bar1).map(_.topicPartition()).toSet
- assertEquals(expected,
- LogManager.findStrayReplicas(0,
- createLeaderAndIsrRequestForStrayDetection(present),
- onDisk.map(mockLog)).toSet)
- }
-
/**
* Test LogManager takes file lock by default and the lock is released after shutdown.
*/
@@ -1475,40 +1431,4 @@ object LogManagerTest {
topics.foreach { t => retval = retval.including(t) }
retval
}
-
- def createLeaderAndIsrRequestForStrayDetection(
- partitions: Iterable[TopicIdPartition],
- leaders: Iterable[Int] = Seq(),
- ): LeaderAndIsrRequest = {
- val nextLeaderIter = leaders.iterator
- def nextLeader(): Int = {
- if (nextLeaderIter.hasNext) {
- nextLeaderIter.next()
- } else {
- 3
- }
- }
- val data = new LeaderAndIsrRequestData().
- setControllerId(1000).
- setIsKRaftController(true).
- setType(AbstractControlRequest.Type.FULL.toByte)
- val topics = new java.util.LinkedHashMap[String, LeaderAndIsrTopicState]
- partitions.foreach(partition => {
- val topicState = topics.computeIfAbsent(partition.topic(),
- _ => new LeaderAndIsrTopicState().
- setTopicId(partition.topicId()).
- setTopicName(partition.topic()))
- topicState.partitionStates().add(new LeaderAndIsrRequestData.LeaderAndIsrPartitionState().
- setTopicName(partition.topic()).
- setPartitionIndex(partition.partition()).
- setControllerEpoch(123).
- setLeader(nextLeader()).
- setLeaderEpoch(456).
- setIsr(java.util.Arrays.asList(3, 4, 5)).
- setReplicas(java.util.Arrays.asList(3, 4, 5)).
- setLeaderRecoveryState(LeaderRecoveryState.RECOVERED.value()))
- })
- data.topicStates().addAll(topics.values())
- new LeaderAndIsrRequest(data, 7.toShort)
- }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/scala/unit/kafka/network/ProcessorTest.scala b/core/src/test/scala/unit/kafka/network/ProcessorTest.scala
index d42ae11bae6..94e93d4d2a8 100644
--- a/core/src/test/scala/unit/kafka/network/ProcessorTest.scala
+++ b/core/src/test/scala/unit/kafka/network/ProcessorTest.scala
@@ -20,10 +20,11 @@ package kafka.network
import kafka.server.SimpleApiVersionManager
import org.apache.kafka.common.errors.{InvalidRequestException, UnsupportedVersionException}
import org.apache.kafka.common.message.ApiMessageType.ListenerType
+import org.apache.kafka.common.message.RequestHeaderData
import org.apache.kafka.common.protocol.ApiKeys
import org.apache.kafka.common.requests.{RequestHeader, RequestTestUtils}
import org.apache.kafka.server.common.{FinalizedFeatures, MetadataVersion}
-import org.junit.jupiter.api.Assertions.assertThrows
+import org.junit.jupiter.api.Assertions.{assertThrows, assertTrue}
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.function.Executable
@@ -32,13 +33,33 @@ import java.util.Collections
class ProcessorTest {
@Test
- def testParseRequestHeaderWithDisabledApi(): Unit = {
+ def testParseRequestHeaderWithDisabledApiVersion(): Unit = {
val requestHeader = RequestTestUtils.serializeRequestHeader(
new RequestHeader(ApiKeys.INIT_PRODUCER_ID, 0, "clientid", 0))
val apiVersionManager = new SimpleApiVersionManager(ListenerType.CONTROLLER, true,
() => new FinalizedFeatures(MetadataVersion.latestTesting(), Collections.emptyMap[String, java.lang.Short], 0, true))
- assertThrows(classOf[InvalidRequestException], (() => Processor.parseRequestHeader(apiVersionManager, requestHeader)): Executable,
+ val e = assertThrows(classOf[InvalidRequestException],
+ (() => Processor.parseRequestHeader(apiVersionManager, requestHeader)): Executable,
"INIT_PRODUCER_ID with listener type CONTROLLER should throw InvalidRequestException exception")
+ assertTrue(e.toString.contains("disabled api"));
+ }
+
+ @Test
+ def testParseRequestHeaderWithUnsupportedApi(): Unit = {
+ // We have to use `RequestHeaderData` since `ApiMessageType` doesn't support this protocol api
+ val headerVersion = 0.toShort
+ val requestHeaderData = new RequestHeaderData()
+ .setRequestApiKey(ApiKeys.LEADER_AND_ISR.id)
+ .setRequestApiVersion(headerVersion)
+ .setClientId("clientid")
+ .setCorrelationId(0);
+ val requestHeader = RequestTestUtils.serializeRequestHeader(new RequestHeader(requestHeaderData, headerVersion))
+ val apiVersionManager = new SimpleApiVersionManager(ListenerType.BROKER, true,
+ () => new FinalizedFeatures(MetadataVersion.latestTesting(), Collections.emptyMap[String, java.lang.Short], 0, true))
+ val e = assertThrows(classOf[InvalidRequestException],
+ (() => Processor.parseRequestHeader(apiVersionManager, requestHeader)): Executable,
+ "LEADER_AND_ISR should throw InvalidRequestException exception")
+ assertTrue(e.toString.contains("Unsupported api"));
}
@Test
@@ -47,8 +68,10 @@ class ProcessorTest {
new RequestHeader(ApiKeys.PRODUCE, 0, "clientid", 0))
val apiVersionManager = new SimpleApiVersionManager(ListenerType.BROKER, true,
() => new FinalizedFeatures(MetadataVersion.latestTesting(), Collections.emptyMap[String, java.lang.Short], 0, true))
- assertThrows(classOf[UnsupportedVersionException], (() => Processor.parseRequestHeader(apiVersionManager, requestHeader)): Executable,
+ val e = assertThrows(classOf[UnsupportedVersionException],
+ (() => Processor.parseRequestHeader(apiVersionManager, requestHeader)): Executable,
"PRODUCE v0 should throw UnsupportedVersionException exception")
+ assertTrue(e.toString.contains("unsupported version"));
}
}
diff --git a/core/src/test/scala/unit/kafka/server/AddPartitionsToTxnManagerTest.scala b/core/src/test/scala/unit/kafka/server/AddPartitionsToTxnManagerTest.scala
index 701ff8f079c..a4ea6ceca32 100644
--- a/core/src/test/scala/unit/kafka/server/AddPartitionsToTxnManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/AddPartitionsToTxnManagerTest.scala
@@ -25,11 +25,11 @@ import org.apache.kafka.common.internals.Topic
import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.{AddPartitionsToTxnTopic, AddPartitionsToTxnTopicCollection, AddPartitionsToTxnTransaction, AddPartitionsToTxnTransactionCollection}
import org.apache.kafka.common.message.AddPartitionsToTxnResponseData
import org.apache.kafka.common.message.AddPartitionsToTxnResponseData.AddPartitionsToTxnResultCollection
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState
import org.apache.kafka.common.{Node, TopicPartition}
import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.requests.{AbstractResponse, AddPartitionsToTxnRequest, AddPartitionsToTxnResponse, MetadataResponse}
import org.apache.kafka.common.utils.MockTime
+import org.apache.kafka.metadata.LeaderAndIsr
import org.apache.kafka.server.metrics.KafkaMetricsGroup
import org.apache.kafka.server.util.RequestAndCompletionHandler
import org.junit.jupiter.api.Assertions.assertEquals
@@ -41,6 +41,7 @@ import org.mockito.ArgumentMatchers.{any, anyLong, anyString}
import org.mockito.MockedConstruction.Context
import org.mockito.Mockito.{mock, mockConstruction, times, verify, verifyNoMoreInteractions, when}
+import java.util
import java.util.concurrent.TimeUnit
import scala.collection.mutable
import scala.jdk.CollectionConverters._
@@ -218,7 +219,7 @@ class AddPartitionsToTxnManagerTest {
}
// The transaction state topic does not exist.
- when(metadataCache.getPartitionInfo(Topic.TRANSACTION_STATE_TOPIC_NAME, 0))
+ when(metadataCache.getLeaderAndIsr(Topic.TRANSACTION_STATE_TOPIC_NAME, 0))
.thenReturn(Option.empty)
checkError()
@@ -403,12 +404,8 @@ class AddPartitionsToTxnManagerTest {
}
private def mockTransactionStateMetadata(partitionIndex: Int, leaderId: Int, leaderNode: Option[Node]): Unit = {
- when(metadataCache.getPartitionInfo(Topic.TRANSACTION_STATE_TOPIC_NAME, partitionIndex))
- .thenReturn(Some(
- new UpdateMetadataPartitionState()
- .setTopicName(Topic.TRANSACTION_STATE_TOPIC_NAME)
- .setPartitionIndex(partitionIndex)
- .setLeader(leaderId)))
+ when(metadataCache.getLeaderAndIsr(Topic.TRANSACTION_STATE_TOPIC_NAME, partitionIndex))
+ .thenReturn(Some(new LeaderAndIsr(leaderId, util.Arrays.asList(leaderId))))
if (leaderId != MetadataResponse.NO_LEADER_ID) {
when(metadataCache.getAliveBrokerNode(leaderId, config.interBrokerListenerName))
.thenReturn(leaderNode)
diff --git a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
index e838fd6f39b..4d89b0ae423 100644
--- a/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/DeleteTopicsRequestTest.scala
@@ -49,8 +49,8 @@ class DeleteTopicsRequestTest extends BaseRequestTest with Logging {
// Ensure one topic partition is offline.
TestUtils.waitUntilTrue(() => {
- aliveBrokers.head.metadataCache.getPartitionInfo(onlineTopic, 0).exists(_.leader() == 1) &&
- aliveBrokers.head.metadataCache.getPartitionInfo(offlineTopic, 0).exists(_.leader() ==
+ aliveBrokers.head.metadataCache.getLeaderAndIsr(onlineTopic, 0).exists(_.leader() == 1) &&
+ aliveBrokers.head.metadataCache.getLeaderAndIsr(offlineTopic, 0).exists(_.leader() ==
MetadataResponse.NO_LEADER_ID)
}, "Topic partition is not offline")
diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
index 46ee16f22ad..355f1c374ba 100644
--- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
+++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala
@@ -2234,7 +2234,7 @@ class KafkaApisTest extends Logging {
when(clientQuotaManager.maybeRecordAndGetThrottleTimeMs(
any[RequestChannel.Request](), anyDouble, anyLong)).thenReturn(0)
when(metadataCache.contains(tp)).thenAnswer(_ => true)
- when(metadataCache.getPartitionInfo(tp.topic(), tp.partition())).thenAnswer(_ => Option.empty)
+ when(metadataCache.getLeaderAndIsr(tp.topic(), tp.partition())).thenAnswer(_ => Option.empty)
when(metadataCache.getAliveBrokerNode(any(), any())).thenReturn(Option.empty)
kafkaApis = createKafkaApis()
kafkaApis.handleProduceRequest(request, RequestLocal.withThreadConfinedCaching)
diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
index 669210f8fcc..91062414cf6 100644
--- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
+++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala
@@ -784,7 +784,7 @@ class MetadataCacheTest {
@ParameterizedTest
@MethodSource(Array("cacheProvider"))
- def testGetPartitionInfo(cache: MetadataCache): Unit = {
+ def testGetLeaderAndIsr(cache: MetadataCache): Unit = {
val topic = "topic"
val topicId = Uuid.randomUuid()
val partitionIndex = 0
@@ -819,14 +819,12 @@ class MetadataCacheTest {
MetadataCacheTest.updateCache(cache, brokers ++ topicRecords ++ partitionStates)
- val partitionState = cache.getPartitionInfo(topic, partitionIndex).get
- assertEquals(topic, partitionState.topicName())
- assertEquals(partitionIndex, partitionState.partitionIndex())
- assertEquals(-1, partitionState.controllerEpoch())
- assertEquals(leader, partitionState.leader())
- assertEquals(leaderEpoch, partitionState.leaderEpoch())
- assertEquals(isr, partitionState.isr())
- assertEquals(replicas, partitionState.replicas())
+ val leaderAndIsr = cache.getLeaderAndIsr(topic, partitionIndex)
+ assertEquals(Some(leader), leaderAndIsr.map(_.leader()))
+ assertEquals(Some(leaderEpoch), leaderAndIsr.map(_.leaderEpoch()))
+ assertEquals(Some(isr), leaderAndIsr.map(_.isr()))
+ assertEquals(Some(-1), leaderAndIsr.map(_.partitionEpoch()))
+ assertEquals(Some(LeaderRecoveryState.RECOVERED), leaderAndIsr.map(_.leaderRecoveryState()))
}
@Test
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 6370aa546d6..2c43537cd02 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -20,11 +20,12 @@ package kafka.server
import com.yammer.metrics.core.{Gauge, Meter, Timer}
import kafka.cluster.PartitionTest.MockPartitionListener
import kafka.cluster.Partition
-import kafka.log.{LogManager, LogManagerTest, UnifiedLog}
+import kafka.log.{LogManager, UnifiedLog}
import kafka.log.remote.RemoteLogManager
import org.apache.kafka.server.log.remote.quota.RLMQuotaManagerConfig.INACTIVE_SENSOR_EXPIRATION_TIME_SECONDS
import org.apache.kafka.server.log.remote.quota.RLMQuotaMetrics
import kafka.server.QuotaFactory.{QuotaManagers, UNBOUNDED_QUOTA}
+import kafka.server.ReplicaManager.StopReplicaPartitionState
import kafka.server.epoch.util.MockBlockingSender
import kafka.server.share.DelayedShareFetch
import kafka.utils.TestUtils.waitUntilTrue
@@ -35,10 +36,8 @@ import org.apache.kafka.common.compress.Compression
import org.apache.kafka.common.config.TopicConfig
import org.apache.kafka.common.errors.{InvalidPidMappingException, KafkaStorageException}
import org.apache.kafka.common.internals.Topic
-import org.apache.kafka.common.message.{DeleteRecordsResponseData, LeaderAndIsrRequestData}
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
+import org.apache.kafka.common.message.DeleteRecordsResponseData
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset
-import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaPartitionState
import org.apache.kafka.common.metadata.{PartitionChangeRecord, PartitionRecord, RemoveTopicRecord, TopicRecord}
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.network.ListenerName
@@ -292,8 +291,8 @@ class ReplicaManagerTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints.asJava), None)
- rm.becomeLeaderOrFollower(0, new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ rm.becomeLeaderOrFollower(0, new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -356,8 +355,8 @@ class ReplicaManagerTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints.asJava), Option.apply(uuid))
- val response = rm.becomeLeaderOrFollower(0, new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val response = rm.becomeLeaderOrFollower(0, new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -430,8 +429,8 @@ class ReplicaManagerTest {
partition.createLogIfNotExists(isNew = false, isFutureReplica = false,
new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints.asJava), None)
// Make this replica the leader.
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -453,8 +452,8 @@ class ReplicaManagerTest {
}
// Make this replica the follower
- val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -528,8 +527,8 @@ class ReplicaManagerTest {
.createLogIfNotExists(isNew = false, isFutureReplica = false,
new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava), None)
- def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -590,8 +589,8 @@ class ReplicaManagerTest {
new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava), None)
// Make this replica the leader.
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -655,8 +654,8 @@ class ReplicaManagerTest {
// Make this replica the leader for the partitions.
Seq(0, 1).foreach { partition =>
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(partition)
.setControllerEpoch(0)
@@ -668,8 +667,7 @@ class ReplicaManagerTest {
.setIsNew(true)).asJava,
Collections.singletonMap(topic, Uuid.randomUuid()),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava,
- false,
- AbstractControlRequest.Type.UNKNOWN
+ LeaderAndIsrRequest.Type.UNKNOWN
).build()
replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
replicaManager.getPartitionOrException(new TopicPartition(topic, partition))
@@ -741,8 +739,8 @@ class ReplicaManagerTest {
// Make this replica the leader.
val brokerList = Seq[Integer](0, 1, 2).asJava
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -801,8 +799,8 @@ class ReplicaManagerTest {
new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava), None)
// Make this replica the leader.
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -922,8 +920,8 @@ class ReplicaManagerTest {
new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava), None)
// Make this replica the leader.
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1007,8 +1005,8 @@ class ReplicaManagerTest {
new LazyOffsetCheckpoints(rm.highWatermarkCheckpoints.asJava), None)
// Make this replica the leader.
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1069,7 +1067,7 @@ class ReplicaManagerTest {
val replicas = aliveBrokersIds.toList.map(Int.box).asJava
// Broker 0 becomes leader of the partition
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1079,7 +1077,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(replicas)
.setIsNew(true)
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState).asJava,
Collections.singletonMap(topic, topicId),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
@@ -1168,7 +1166,7 @@ class ReplicaManagerTest {
val replicas = aliveBrokersIds.toList.map(Int.box).asJava
// Broker 0 becomes leader of the partition
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1178,7 +1176,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(replicas)
.setIsNew(true)
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState).asJava,
Collections.singletonMap(topic, topicId),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
@@ -1229,7 +1227,7 @@ class ReplicaManagerTest {
val tidp2 = new TopicIdPartition(Uuid.randomUuid(), tp2)
// Broker 0 becomes leader of the partition
- val leaderAndIsrPartitionState2 = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState2 = new LeaderAndIsrRequest.PartitionState()
.setTopicName("noIdTopic")
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1239,7 +1237,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(replicas)
.setIsNew(true)
- val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState2).asJava,
Collections.emptyMap(),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
@@ -1298,9 +1296,9 @@ class ReplicaManagerTest {
val partition1Replicas = Seq[Integer](0, 2).asJava
val topicIds = Map(tp0.topic -> topicId, tp1.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -1310,7 +1308,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(partition0Replicas)
.setIsNew(true),
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp1.topic)
.setPartitionIndex(tp1.partition)
.setControllerEpoch(0)
@@ -1424,7 +1422,7 @@ class ReplicaManagerTest {
// Make local partition a follower - because epoch increased by more than 1, truncation should
// trigger even though leader does not change
leaderEpoch += leaderEpochIncrement
- val leaderAndIsrRequest0 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
+ val leaderAndIsrRequest0 = new LeaderAndIsrRequest.Builder(
controllerId, controllerEpoch, brokerEpoch,
Seq(leaderAndIsrPartitionState(tp, leaderEpoch, leaderBrokerId, aliveBrokerIds)).asJava,
Collections.singletonMap(topic, topicId),
@@ -1505,8 +1503,8 @@ class ReplicaManagerTest {
val tidp0 = new TopicIdPartition(topicId, tp0)
// Make this replica the follower
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1559,8 +1557,8 @@ class ReplicaManagerTest {
val tidp0 = new TopicIdPartition(topicId, tp0)
// Make this replica the leader
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1616,11 +1614,10 @@ class ReplicaManagerTest {
// Make this replica the leader and remove follower from ISR.
val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(
- ApiKeys.LEADER_AND_ISR.latestVersion,
0,
0,
brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1684,8 +1681,8 @@ class ReplicaManagerTest {
val tidp0 = new TopicIdPartition(topicId, tp0)
// Make this replica the follower
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1743,8 +1740,8 @@ class ReplicaManagerTest {
// Make this replica the leader
val leaderEpoch = 1
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1810,8 +1807,8 @@ class ReplicaManagerTest {
val tidp0 = new TopicIdPartition(topicId, tp0)
// Make this replica the follower
- val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -1910,8 +1907,8 @@ class ReplicaManagerTest {
val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava)
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -1955,8 +1952,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2004,8 +2001,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2025,8 +2022,8 @@ class ReplicaManagerTest {
assertFalse(fetchResult.hasFired)
// Become a follower and ensure that the delayed fetch returns immediately
- val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2057,8 +2054,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2085,8 +2082,8 @@ class ReplicaManagerTest {
assertFalse(fetchResult.hasFired)
// Become a follower and ensure that the delayed fetch returns immediately
- val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeFollowerRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2116,8 +2113,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2162,8 +2159,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2186,10 +2183,7 @@ class ReplicaManagerTest {
// We have a fetch in purgatory, now receive a stop replica request and
// assert that the fetch returns with a NOT_LEADER error
replicaManager.stopReplicas(2, 0, 0,
- mutable.Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setDeletePartition(true)
- .setLeaderEpoch(LeaderAndIsr.EPOCH_DURING_DELETE)))
+ mutable.Map(tp0 -> StopReplicaPartitionState(LeaderAndIsr.EPOCH_DURING_DELETE, true)))
assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, fetchResult.assertFired.error)
} finally {
@@ -2208,8 +2202,8 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -2229,10 +2223,7 @@ class ReplicaManagerTest {
when(replicaManager.metadataCache.contains(tp0)).thenReturn(true)
replicaManager.stopReplicas(2, 0, 0,
- mutable.Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setDeletePartition(true)
- .setLeaderEpoch(LeaderAndIsr.EPOCH_DURING_DELETE)))
+ mutable.Map(tp0 -> new StopReplicaPartitionState(LeaderAndIsr.EPOCH_DURING_DELETE, true)))
assertNotNull(produceResult.get)
assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, produceResult.get.error)
@@ -2689,157 +2680,6 @@ class ReplicaManagerTest {
}
}
- @Test
- def testFullLeaderAndIsrStrayPartitions(): Unit = {
- val logManager = TestUtils.createLogManager(config.logDirs.map(new File(_)), defaultConfig = new LogConfig(new Properties()), time = time)
- val quotaManager = QuotaFactory.instantiate(config, metrics, time, "")
- val replicaManager = new ReplicaManager(
- metrics = metrics,
- config = config,
- time = time,
- scheduler = time.scheduler,
- logManager = logManager,
- quotaManagers = quotaManager,
- metadataCache = MetadataCache.kRaftMetadataCache(config.brokerId, () => KRaftVersion.KRAFT_VERSION_0),
- logDirFailureChannel = new LogDirFailureChannel(config.logDirs.size),
- alterPartitionManager = alterPartitionManager,
- threadNamePrefix = Option(this.getClass.getName))
-
- try {
-
- logManager.startup(Set.empty[String])
-
- // Create a hosted topic, a hosted topic that will become stray
- createHostedLogs("hosted-topic", numLogs = 2, replicaManager).toSet
- createHostedLogs("hosted-stray", numLogs = 10, replicaManager).toSet
-
- val lisr = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
- 3000, 0, brokerEpoch,
- Seq(
- new LeaderAndIsrPartitionState()
- .setTopicName("hosted-topic")
- .setPartitionIndex(0)
- .setControllerEpoch(controllerEpoch)
- .setLeader(0)
- .setLeaderEpoch(10)
- .setIsr(Seq[Integer](0, 1).asJava)
- .setPartitionEpoch(0)
- .setReplicas(Seq[Integer](0, 1).asJava)
- .setIsNew(false),
- new LeaderAndIsrPartitionState()
- .setTopicName("hosted-topic")
- .setPartitionIndex(1)
- .setControllerEpoch(controllerEpoch)
- .setLeader(1)
- .setLeaderEpoch(10)
- .setIsr(Seq[Integer](1, 0).asJava)
- .setPartitionEpoch(0)
- .setReplicas(Seq[Integer](1, 0).asJava)
- .setIsNew(false)
- ).asJava,
- topicIds.asJava,
- Set(new Node(0, "host0", 0), new Node(1, "host1", 1)).asJava,
- true,
- AbstractControlRequest.Type.FULL
- ).build()
-
- replicaManager.becomeLeaderOrFollower(0, lisr, (_, _) => ())
-
- val ht0 = replicaManager.getPartition(new TopicPartition("hosted-topic", 0))
- assertTrue(ht0.isInstanceOf[HostedPartition.Online])
-
- val stray0 = replicaManager.getPartition(new TopicPartition("hosted-stray", 0))
-
- assertTrue(stray0.isInstanceOf[HostedPartition.Online])
- } finally {
- Utils.tryAll(util.Arrays.asList[Callable[Void]] (
- () => {
- replicaManager.shutdown(checkpointHW = false)
- null
- },
- () => {
- logManager.shutdown()
- null
- },
- () => {
- quotaManager.shutdown()
- null
- }
- ))
- }
- }
-
- @Test
- def testUpdateStrayLogs(): Unit = {
- val logManager = TestUtils.createLogManager(config.logDirs.map(new File(_)), defaultConfig = new LogConfig(new Properties()), time = time)
- val quotaManager = QuotaFactory.instantiate(config, metrics, time, "")
- val replicaManager = new ReplicaManager(
- metrics = metrics,
- config = config,
- time = time,
- scheduler = time.scheduler,
- logManager = logManager,
- quotaManagers = quotaManager,
- metadataCache = MetadataCache.kRaftMetadataCache(config.brokerId, () => KRaftVersion.KRAFT_VERSION_0),
- logDirFailureChannel = new LogDirFailureChannel(config.logDirs.size),
- alterPartitionManager = alterPartitionManager,
- threadNamePrefix = Option(this.getClass.getName))
-
- try {
- logManager.startup(Set.empty[String])
-
- // Create a hosted topic, a hosted topic that will become stray, and a stray topic
- val validLogs = createHostedLogs("hosted-topic", numLogs = 2, replicaManager).toSet
- createHostedLogs("hosted-stray", numLogs = 10, replicaManager).toSet
- createStrayLogs(10, logManager)
-
- val allReplicasFromLISR = Set(
- new TopicPartition("hosted-topic", 0),
- new TopicPartition("hosted-topic", 1)
- ).map(p => new TopicIdPartition(new Uuid(p.topic().hashCode, p.topic().hashCode), p))
-
- replicaManager.updateStrayLogs(
- LogManager.findStrayReplicas(
- config.nodeId,
- LogManagerTest.createLeaderAndIsrRequestForStrayDetection(allReplicasFromLISR),
- logManager.allLogs))
-
- assertEquals(validLogs, logManager.allLogs.toSet)
- assertEquals(validLogs.size, replicaManager.partitionCount.value)
- } finally {
- Utils.tryAll(util.Arrays.asList[Callable[Void]](
- () => {
- replicaManager.shutdown(checkpointHW = false)
- null
- },
- () => {
- logManager.shutdown()
- null
- },
- () => {
- quotaManager.shutdown()
- null
- }
- ))
- }
- }
-
- private def createHostedLogs(name: String, numLogs: Int, replicaManager: ReplicaManager): Seq[UnifiedLog] = {
- for (i <- 0 until numLogs) yield {
- val topicPartition = new TopicPartition(name, i)
- val partition = replicaManager.createPartition(topicPartition)
- partition.createLogIfNotExists(isNew = true, isFutureReplica = false,
- new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava), topicId = Some(new Uuid(name.hashCode, name.hashCode)))
- partition.log.get
- }
- }
-
- private def createStrayLogs(numLogs: Int, logManager: LogManager): Seq[UnifiedLog] = {
- val name = "stray"
- for (i <- 0 until numLogs)
- yield logManager.getOrCreateLog(new TopicPartition(name, i), topicId = Some(new Uuid(name.hashCode, name.hashCode)))
- }
-
private def sendProducerAppend(
replicaManager: ReplicaManager,
topicPartition: TopicPartition,
@@ -3054,8 +2894,8 @@ class ReplicaManagerTest {
leaderEpoch: Int,
leaderBrokerId: Int,
aliveBrokerIds: Seq[Integer],
- isNew: Boolean = false): LeaderAndIsrPartitionState = {
- new LeaderAndIsrPartitionState()
+ isNew: Boolean = false): LeaderAndIsrRequest.PartitionState = {
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(topicPartition.partition)
.setControllerEpoch(controllerEpoch)
@@ -3542,10 +3382,9 @@ class ReplicaManagerTest {
val partition1Replicas = Seq[Integer](1, 0).asJava
val topicIds = Map(tp0.topic -> Uuid.randomUuid(), tp1.topic -> Uuid.randomUuid()).asJava
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
- controllerId, 0, brokerEpoch,
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(controllerId, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(controllerEpoch)
@@ -3555,7 +3394,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(partition0Replicas)
.setIsNew(true),
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp1.topic)
.setPartitionIndex(tp1.partition)
.setControllerEpoch(controllerEpoch)
@@ -3573,10 +3412,9 @@ class ReplicaManagerTest {
rm1.becomeLeaderOrFollower(correlationId, leaderAndIsrRequest1, (_, _) => ())
// make broker 0 the leader of partition 1 so broker 1 loses its leadership position
- val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
- controllerEpoch, brokerEpoch,
+ val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder( controllerId, controllerEpoch, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(controllerEpoch)
@@ -3586,7 +3424,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(partition0Replicas)
.setIsNew(true),
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp1.topic)
.setPartitionIndex(tp1.partition)
.setControllerEpoch(controllerEpoch)
@@ -3638,10 +3476,9 @@ class ReplicaManagerTest {
val partition1Replicas = Seq[Integer](1, 0).asJava
val topicIds = Map(tp0.topic -> Uuid.randomUuid(), tp1.topic -> Uuid.randomUuid()).asJava
- val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion,
- controllerId, 0, brokerEpoch,
+ val leaderAndIsrRequest1 = new LeaderAndIsrRequest.Builder(controllerId, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(controllerEpoch)
@@ -3651,7 +3488,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(partition0Replicas)
.setIsNew(true),
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp1.topic)
.setPartitionIndex(tp1.partition)
.setControllerEpoch(controllerEpoch)
@@ -3669,10 +3506,10 @@ class ReplicaManagerTest {
rm1.becomeLeaderOrFollower(correlationId, leaderAndIsrRequest1, (_, _) => ())
// make broker 0 the leader of partition 1 so broker 1 loses its leadership position
- val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, controllerId,
+ val leaderAndIsrRequest2 = new LeaderAndIsrRequest.Builder(controllerId,
controllerEpoch, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(controllerEpoch)
@@ -3682,7 +3519,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(partition0Replicas)
.setIsNew(true),
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp1.topic)
.setPartitionIndex(tp1.partition)
.setControllerEpoch(controllerEpoch)
@@ -3777,7 +3614,7 @@ class ReplicaManagerTest {
val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava)
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 10, brokerEpoch,
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 10, brokerEpoch,
Seq(leaderAndIsrPartitionState(tp0, 1, 0, Seq(0, 1), true)).asJava,
Collections.singletonMap(topic, Uuid.randomUuid()),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava
@@ -3785,12 +3622,7 @@ class ReplicaManagerTest {
replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
- val partitionStates = Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setLeaderEpoch(1)
- .setDeletePartition(false)
- )
-
+ val partitionStates = Map(tp0 -> StopReplicaPartitionState(1, false))
val (_, error) = replicaManager.stopReplicas(1, 0, 0, partitionStates)
assertEquals(Errors.STALE_CONTROLLER_EPOCH, error)
if (enableRemoteStorage) {
@@ -3813,7 +3645,7 @@ class ReplicaManagerTest {
val offsetCheckpoints = new LazyOffsetCheckpoints(replicaManager.highWatermarkCheckpoints.asJava)
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState(tp0, 1, 0, Seq(0, 1), true)).asJava,
Collections.singletonMap(topic, Uuid.randomUuid()),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava
@@ -3822,12 +3654,7 @@ class ReplicaManagerTest {
replicaManager.becomeLeaderOrFollower(1, becomeLeaderRequest, (_, _) => ())
replicaManager.markPartitionOffline(tp0)
- val partitionStates = Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setLeaderEpoch(1)
- .setDeletePartition(false)
- )
-
+ val partitionStates = Map(tp0 -> StopReplicaPartitionState(1, false))
val (result, error) = replicaManager.stopReplicas(1, 0, 0, partitionStates)
assertEquals(Errors.NONE, error)
assertEquals(Map(tp0 -> Errors.KAFKA_STORAGE_ERROR), result)
@@ -3875,12 +3702,7 @@ class ReplicaManagerTest {
Files.createFile(dir.toPath)
}
- val partitionStates = Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setLeaderEpoch(1)
- .setDeletePartition(deletePartitions)
- )
-
+ val partitionStates = Map(tp0 -> new StopReplicaPartitionState(1, deletePartitions))
val (result, error) = replicaManager.stopReplicas(1, 0, 0, partitionStates)
assertEquals(Errors.NONE, error)
@@ -3987,9 +3809,9 @@ class ReplicaManagerTest {
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4042,9 +3864,9 @@ class ReplicaManagerTest {
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4134,9 +3956,9 @@ class ReplicaManagerTest {
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4247,9 +4069,9 @@ class ReplicaManagerTest {
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4346,9 +4168,9 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4399,9 +4221,9 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4458,9 +4280,9 @@ class ReplicaManagerTest {
replicaManager.createPartition(tp0).createLogIfNotExists(isNew = false, isFutureReplica = false, offsetCheckpoints, None)
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp0.topic -> topicId).asJava
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp0.topic)
.setPartitionIndex(tp0.partition)
.setControllerEpoch(0)
@@ -4552,7 +4374,7 @@ class ReplicaManagerTest {
logDirFailureChannel).read()
}
- val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val becomeLeaderRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState(tp0, 1, 0, Seq(0, 1), true)).asJava,
Collections.singletonMap(tp0.topic(), Uuid.randomUuid()),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava
@@ -4579,12 +4401,7 @@ class ReplicaManagerTest {
partition.log.get.partitionMetadataFile = Some(mockPartitionMetadataFile)
}
- val partitionStates = Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setLeaderEpoch(leaderEpoch)
- .setDeletePartition(deletePartition)
- )
-
+ val partitionStates = Map(tp0 -> StopReplicaPartitionState(leaderEpoch, deletePartition))
val (result, error) = replicaManager.stopReplicas(1, 0, 0, partitionStates)
assertEquals(Errors.NONE, error)
assertEquals(Map(tp0 -> expectedOutput), result)
@@ -4656,8 +4473,8 @@ class ReplicaManagerTest {
val topicNames = topicIds.asScala.map(_.swap).asJava
def leaderAndIsrRequest(epoch: Int, topicIds: java.util.Map[String, Uuid]): LeaderAndIsrRequest =
- new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -4703,8 +4520,8 @@ class ReplicaManagerTest {
val topicIds = Collections.singletonMap(topic, Uuid.randomUuid())
val topicNames = topicIds.asScala.map(_.swap).asJava
- def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ def leaderAndIsrRequest(epoch: Int): LeaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -4733,67 +4550,6 @@ class ReplicaManagerTest {
}
}
- @Test
- def testPartitionMetadataFileCreatedAfterPreviousRequestWithoutIds(): Unit = {
- val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer(time))
- try {
- val brokerList = Seq[Integer](0, 1).asJava
- val topicPartition = new TopicPartition(topic, 0)
- val topicPartition2 = new TopicPartition(topic, 1)
-
- def leaderAndIsrRequest(topicIds: util.Map[String, Uuid], version: Short, partition: Int = 0, leaderEpoch: Int = 0): LeaderAndIsrRequest =
- new LeaderAndIsrRequest.Builder(version, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
- .setTopicName(topic)
- .setPartitionIndex(partition)
- .setControllerEpoch(0)
- .setLeader(0)
- .setLeaderEpoch(leaderEpoch)
- .setIsr(brokerList)
- .setPartitionEpoch(0)
- .setReplicas(brokerList)
- .setIsNew(true)).asJava,
- topicIds,
- Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
-
- // Send a request without a topic ID so that we have a log without a topic ID associated to the partition.
- val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(Collections.emptyMap(), 4), (_, _) => ())
- assertEquals(Errors.NONE, response.partitionErrors(Collections.emptyMap()).get(topicPartition))
- assertTrue(replicaManager.localLog(topicPartition).isDefined)
- val log = replicaManager.localLog(topicPartition).get
- assertFalse(log.partitionMetadataFile.get.exists())
- assertTrue(log.topicId.isEmpty)
-
- val response2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(topicIds.asJava, ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ())
- assertEquals(Errors.NONE, response2.partitionErrors(topicNames.asJava).get(topicPartition))
- assertTrue(replicaManager.localLog(topicPartition).isDefined)
- assertTrue(log.partitionMetadataFile.get.exists())
- assertTrue(log.topicId.isDefined)
- assertEquals(topicId, log.topicId.get)
-
- // Repeat with partition 2, but in this case, update the leader epoch
- // Send a request without a topic ID so that we have a log without a topic ID associated to the partition.
- val response3 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(Collections.emptyMap(), 4, 1), (_, _) => ())
- assertEquals(Errors.NONE, response3.partitionErrors(Collections.emptyMap()).get(topicPartition2))
- assertTrue(replicaManager.localLog(topicPartition2).isDefined)
- val log2 = replicaManager.localLog(topicPartition2).get
- assertFalse(log2.partitionMetadataFile.get.exists())
- assertTrue(log2.topicId.isEmpty)
-
- val response4 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(topicIds.asJava, ApiKeys.LEADER_AND_ISR.latestVersion, 1, 1), (_, _) => ())
- assertEquals(Errors.NONE, response4.partitionErrors(topicNames.asJava).get(topicPartition2))
- assertTrue(replicaManager.localLog(topicPartition2).isDefined)
- assertTrue(log2.partitionMetadataFile.get.exists())
- assertTrue(log2.topicId.isDefined)
- assertEquals(topicId, log2.topicId.get)
-
- assertEquals(topicId, log.partitionMetadataFile.get.read().topicId)
- assertEquals(topicId, log2.partitionMetadataFile.get.read().topicId)
- } finally {
- replicaManager.shutdown(checkpointHW = false)
- }
- }
-
@Test
def testInconsistentIdReturnsError(): Unit = {
val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer(time))
@@ -4807,8 +4563,8 @@ class ReplicaManagerTest {
val invalidTopicNames = invalidTopicIds.asScala.map(_.swap).asJava
def leaderAndIsrRequest(epoch: Int, topicIds: java.util.Map[String, Uuid]): LeaderAndIsrRequest =
- new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -4844,53 +4600,39 @@ class ReplicaManagerTest {
try {
val brokerList = Seq[Integer](0, 1).asJava
val topicPartition = new TopicPartition(topic, 0)
- val topicPartitionFoo = new TopicPartition("foo", 0)
val topicPartitionFake = new TopicPartition("fakeTopic", 0)
val topicIds = Map(topic -> Uuid.ZERO_UUID, "foo" -> Uuid.randomUuid()).asJava
val topicNames = topicIds.asScala.map(_.swap).asJava
- def leaderAndIsrRequest(epoch: Int, name: String, version: Short): LeaderAndIsrRequest = LeaderAndIsrRequest.parse(
- new LeaderAndIsrRequest.Builder(version, 0, 0, brokerEpoch,
- Seq(new LeaderAndIsrPartitionState()
- .setTopicName(name)
- .setPartitionIndex(0)
- .setControllerEpoch(0)
- .setLeader(0)
- .setLeaderEpoch(epoch)
- .setIsr(brokerList)
- .setPartitionEpoch(0)
- .setReplicas(brokerList)
- .setIsNew(true)).asJava,
- topicIds,
- Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build().serialize(), version)
+ def leaderAndIsrRequest(epoch: Int, name: String): LeaderAndIsrRequest =
+ new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
+ Seq(new LeaderAndIsrRequest.PartitionState()
+ .setTopicName(name)
+ .setPartitionIndex(0)
+ .setControllerEpoch(0)
+ .setLeader(0)
+ .setLeaderEpoch(epoch)
+ .setIsr(brokerList)
+ .setPartitionEpoch(0)
+ .setReplicas(brokerList)
+ .setIsNew(true)).asJava,
+ topicIds,
+ Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
// There is no file if the topic does not have an associated topic ID.
- val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "fakeTopic", ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ())
+ val response = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "fakeTopic"), (_, _) => ())
assertTrue(replicaManager.localLog(topicPartitionFake).isDefined)
val log = replicaManager.localLog(topicPartitionFake).get
assertFalse(log.partitionMetadataFile.get.exists())
assertEquals(Errors.NONE, response.partitionErrors(topicNames).get(topicPartition))
// There is no file if the topic has the default UUID.
- val response2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, topic, ApiKeys.LEADER_AND_ISR.latestVersion), (_, _) => ())
+ val response2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, topic), (_, _) => ())
assertTrue(replicaManager.localLog(topicPartition).isDefined)
val log2 = replicaManager.localLog(topicPartition).get
assertFalse(log2.partitionMetadataFile.get.exists())
assertEquals(Errors.NONE, response2.partitionErrors(topicNames).get(topicPartition))
- // There is no file if the request an older version
- val response3 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(0, "foo", 0), (_, _) => ())
- assertTrue(replicaManager.localLog(topicPartitionFoo).isDefined)
- val log3 = replicaManager.localLog(topicPartitionFoo).get
- assertFalse(log3.partitionMetadataFile.get.exists())
- assertEquals(Errors.NONE, response3.partitionErrors(topicNames).get(topicPartitionFoo))
-
- // There is no file if the request is an older version
- val response4 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest(1, "foo", 4), (_, _) => ())
- assertTrue(replicaManager.localLog(topicPartitionFoo).isDefined)
- val log4 = replicaManager.localLog(topicPartitionFoo).get
- assertFalse(log4.partitionMetadataFile.get.exists())
- assertEquals(Errors.NONE, response4.partitionErrors(topicNames).get(topicPartitionFoo))
} finally {
replicaManager.shutdown(checkpointHW = false)
}
@@ -4940,10 +4682,9 @@ class ReplicaManagerTest {
isNew: Boolean = true,
brokerEpoch: Int = 0,
controllerId: Int = 0,
- controllerEpoch: Int = 0,
- version: Short = LeaderAndIsrRequestData.HIGHEST_SUPPORTED_VERSION
+ controllerEpoch: Int = 0
): LeaderAndIsrRequest = {
- val partitionState = new LeaderAndIsrPartitionState()
+ val partitionState = new LeaderAndIsrRequest.PartitionState()
.setTopicName(topicPartition.topic)
.setPartitionIndex(topicPartition.partition)
.setControllerEpoch(controllerEpoch)
@@ -4961,7 +4702,6 @@ class ReplicaManagerTest {
val nodes = Set(mkNode(controllerId)) ++ replicas.map(mkNode).toSet
new LeaderAndIsrRequest.Builder(
- version,
controllerId,
controllerEpoch,
brokerEpoch,
@@ -6122,7 +5862,7 @@ class ReplicaManagerTest {
assertFalse(replicaManager.maybeAddListener(tp, listener))
// Broker 0 becomes leader of the partition
- val leaderAndIsrPartitionState = new LeaderAndIsrPartitionState()
+ val leaderAndIsrPartitionState = new LeaderAndIsrRequest.PartitionState()
.setTopicName(topic)
.setPartitionIndex(0)
.setControllerEpoch(0)
@@ -6132,7 +5872,7 @@ class ReplicaManagerTest {
.setPartitionEpoch(0)
.setReplicas(replicas)
.setIsNew(true)
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(leaderAndIsrPartitionState).asJava,
Collections.singletonMap(topic, topicId),
Set(new Node(0, "host1", 0), new Node(1, "host2", 1)).asJava).build()
@@ -6266,45 +6006,9 @@ class ReplicaManagerTest {
assertEquals(expectedTopicId, fetchState.get.topicId)
}
- @ParameterizedTest
- @ValueSource(booleans = Array(true, false))
- def testPartitionFetchStateUpdatesWithTopicIdChanges(startsWithTopicId: Boolean): Unit = {
- val aliveBrokersIds = Seq(0, 1)
- val replicaManager = setupReplicaManagerWithMockedPurgatories(new MockTimer(time),
- brokerId = 0, aliveBrokersIds)
- try {
- val tp = new TopicPartition(topic, 0)
- val leaderAndIsr = new LeaderAndIsr(1, aliveBrokersIds.toList.map(Int.box).asJava)
-
- // This test either starts with a topic ID in the PartitionFetchState and removes it on the next request (startsWithTopicId)
- // or does not start with a topic ID in the PartitionFetchState and adds one on the next request (!startsWithTopicId)
- val startingId = if (startsWithTopicId) topicId else Uuid.ZERO_UUID
- val startingIdOpt = if (startsWithTopicId) Some(topicId) else None
- val leaderAndIsrRequest1 = makeLeaderAndIsrRequest(startingId, tp, aliveBrokersIds, leaderAndIsr)
- val leaderAndIsrResponse1 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest1, (_, _) => ())
- assertEquals(Errors.NONE, leaderAndIsrResponse1.error)
-
- assertFetcherHasTopicId(replicaManager.replicaFetcherManager, tp, startingIdOpt)
-
- val endingId = if (!startsWithTopicId) topicId else Uuid.ZERO_UUID
- val endingIdOpt = if (!startsWithTopicId) Some(topicId) else None
- val leaderAndIsrRequest2 = makeLeaderAndIsrRequest(endingId, tp, aliveBrokersIds, leaderAndIsr)
- val leaderAndIsrResponse2 = replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest2, (_, _) => ())
- assertEquals(Errors.NONE, leaderAndIsrResponse2.error)
-
- assertFetcherHasTopicId(replicaManager.replicaFetcherManager, tp, endingIdOpt)
- } finally {
- replicaManager.shutdown(checkpointHW = false)
- }
- }
-
- @ParameterizedTest
- @ValueSource(booleans = Array(true, false))
- def testReplicaAlterLogDirsWithAndWithoutIds(usesTopicIds: Boolean): Unit = {
+ @Test
+ def testReplicaAlterLogDirs(): Unit = {
val tp = new TopicPartition(topic, 0)
- val version = if (usesTopicIds) LeaderAndIsrRequestData.HIGHEST_SUPPORTED_VERSION else 4.toShort
- val topicId = if (usesTopicIds) this.topicId else Uuid.ZERO_UUID
- val topicIdOpt = if (usesTopicIds) Some(topicId) else None
val mockReplicaAlterLogDirsManager = mock(classOf[ReplicaAlterLogDirsManager])
val replicaManager = setupReplicaManagerWithMockedPurgatories(
@@ -6325,7 +6029,6 @@ class ReplicaManagerTest {
topicPartition = tp,
replicas = Seq(0, 1),
leaderAndIsr = new LeaderAndIsr(0, List(0, 1).map(Int.box).asJava),
- version = version
)
replicaManager.becomeLeaderOrFollower(0, leaderAndIsrRequest, (_, _) => ())
@@ -6336,16 +6039,15 @@ class ReplicaManagerTest {
// Make sure the future log is created with the correct topic ID.
val futureLog = replicaManager.futureLocalLogOrException(tp)
- assertEquals(topicIdOpt, futureLog.topicId)
+ assertEquals(Some(topicId), futureLog.topicId)
// Verify that addFetcherForPartitions was called with the correct topic ID.
- verify(mockReplicaAlterLogDirsManager, times(1))
- .addFetcherForPartitions(Map(tp -> InitialFetchState(
- topicId = topicIdOpt,
- leader = new BrokerEndPoint(0, "localhost", -1),
- currentLeaderEpoch = 0,
- initOffset = 0
- )))
+ verify(mockReplicaAlterLogDirsManager, times(1)).addFetcherForPartitions(Map(tp -> InitialFetchState(
+ topicId = Some(topicId),
+ leader = new BrokerEndPoint(0, "localhost", -1),
+ currentLeaderEpoch = 0,
+ initOffset = 0
+ )))
} finally {
replicaManager.shutdown(checkpointHW = false)
}
@@ -6453,12 +6155,7 @@ class ReplicaManagerTest {
val requestLeaderEpoch = 1
val deleteLocalLog = true
- val partitionStates = Map(tp0 -> new StopReplicaPartitionState()
- .setPartitionIndex(tp0.partition)
- .setLeaderEpoch(requestLeaderEpoch)
- .setDeletePartition(deleteLocalLog)
- )
-
+ val partitionStates = Map(tp0 -> StopReplicaPartitionState(requestLeaderEpoch, deleteLocalLog))
val (result, error) = replicaManager.stopReplicas(1, 0, 0, partitionStates)
assertEquals(Errors.NONE, error)
@@ -6654,9 +6351,9 @@ class ReplicaManagerTest {
val partition0Replicas = Seq[Integer](0, 1).asJava
val topicIds = Map(tp.topic -> topicId).asJava
val leaderEpoch = 0
- val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, 0, 0, brokerEpoch,
+ val leaderAndIsrRequest = new LeaderAndIsrRequest.Builder(0, 0, brokerEpoch,
Seq(
- new LeaderAndIsrPartitionState()
+ new LeaderAndIsrRequest.PartitionState()
.setTopicName(tp.topic)
.setPartitionIndex(tp.partition)
.setControllerEpoch(0)
diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
index 700edbf8d4d..40260e62613 100644
--- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
+++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala
@@ -24,15 +24,11 @@ import org.apache.kafka.common.internals.Topic
import org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic
import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection}
import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState
import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity
import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartition, ListOffsetsTopic}
import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.{OffsetForLeaderPartition, OffsetForLeaderTopic, OffsetForLeaderTopicCollection}
-import org.apache.kafka.common.message.StopReplicaRequestData.{StopReplicaPartitionState, StopReplicaTopicState}
-import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState}
import org.apache.kafka.common.message._
import org.apache.kafka.common.metrics.{KafkaMetric, Quota, Sensor}
-import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.protocol.ApiKeys
import org.apache.kafka.common.quota.ClientQuotaFilter
import org.apache.kafka.common.record._
@@ -41,7 +37,6 @@ import org.apache.kafka.common.resource.{PatternType, ResourceType => AdminResou
import org.apache.kafka.common.security.auth._
import org.apache.kafka.common.utils.{Sanitizer, SecurityUtils}
import org.apache.kafka.coordinator.group.GroupCoordinatorConfig
-import org.apache.kafka.metadata.LeaderAndIsr
import org.apache.kafka.metadata.authorizer.StandardAuthorizer
import org.apache.kafka.network.Session
import org.apache.kafka.server.authorizer.{Action, AuthorizableRequestContext, AuthorizationResult}
@@ -70,7 +65,6 @@ class RequestQuotaTest extends BaseRequestTest {
private val unthrottledClientId = "unthrottled-client"
private val smallQuotaProducerClientId = "small-quota-producer-client"
private val smallQuotaConsumerClientId = "small-quota-consumer-client"
- private val brokerId: Integer = 0
private var leaderNode: KafkaBroker = _
// Run tests concurrently since a throttle could be up to 1 second because quota percentage allocated is very low
@@ -275,61 +269,6 @@ class RequestQuotaTest extends BaseRequestTest {
ListOffsetsRequest.Builder.forConsumer(false, IsolationLevel.READ_UNCOMMITTED)
.setTargetTimes(List(topic).asJava)
- case ApiKeys.LEADER_AND_ISR =>
- new LeaderAndIsrRequest.Builder(ApiKeys.LEADER_AND_ISR.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
- Seq(new LeaderAndIsrPartitionState()
- .setTopicName(tp.topic)
- .setPartitionIndex(tp.partition)
- .setControllerEpoch(Int.MaxValue)
- .setLeader(brokerId)
- .setLeaderEpoch(Int.MaxValue)
- .setIsr(List(brokerId).asJava)
- .setPartitionEpoch(2)
- .setReplicas(Seq(brokerId).asJava)
- .setIsNew(true)).asJava,
- getTopicIds().asJava,
- Set(new Node(brokerId, "localhost", 0)).asJava)
-
- case ApiKeys.STOP_REPLICA =>
- val topicStates = Seq(
- new StopReplicaTopicState()
- .setTopicName(tp.topic())
- .setPartitionStates(Seq(new StopReplicaPartitionState()
- .setPartitionIndex(tp.partition())
- .setLeaderEpoch(LeaderAndIsr.INITIAL_LEADER_EPOCH + 2)
- .setDeletePartition(true)).asJava)
- ).asJava
- new StopReplicaRequest.Builder(ApiKeys.STOP_REPLICA.latestVersion, brokerId,
- Int.MaxValue, Long.MaxValue, false, topicStates)
-
- case ApiKeys.UPDATE_METADATA =>
- val partitionState = Seq(new UpdateMetadataPartitionState()
- .setTopicName(tp.topic)
- .setPartitionIndex(tp.partition)
- .setControllerEpoch(Int.MaxValue)
- .setLeader(brokerId)
- .setLeaderEpoch(Int.MaxValue)
- .setIsr(List(brokerId).asJava)
- .setZkVersion(2)
- .setReplicas(Seq(brokerId).asJava)).asJava
- val securityProtocol = SecurityProtocol.PLAINTEXT
- val brokers = Seq(new UpdateMetadataBroker()
- .setId(brokerId)
- .setEndpoints(Seq(new UpdateMetadataEndpoint()
- .setHost("localhost")
- .setPort(0)
- .setSecurityProtocol(securityProtocol.id)
- .setListener(ListenerName.forSecurityProtocol(securityProtocol).value)).asJava)).asJava
- new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion, brokerId, Int.MaxValue, Long.MaxValue,
- partitionState, brokers, Collections.emptyMap())
-
- case ApiKeys.CONTROLLED_SHUTDOWN =>
- new ControlledShutdownRequest.Builder(
- new ControlledShutdownRequestData()
- .setBrokerId(brokerId)
- .setBrokerEpoch(Long.MaxValue),
- ApiKeys.CONTROLLED_SHUTDOWN.latestVersion)
-
case ApiKeys.OFFSET_COMMIT =>
new OffsetCommitRequest.Builder(
new OffsetCommitRequestData()
diff --git a/core/src/test/scala/unit/kafka/server/ShareGroupHeartbeatRequestTest.scala b/core/src/test/scala/unit/kafka/server/ShareGroupHeartbeatRequestTest.scala
index 71addf733b0..c471b0f2793 100644
--- a/core/src/test/scala/unit/kafka/server/ShareGroupHeartbeatRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ShareGroupHeartbeatRequestTest.scala
@@ -894,7 +894,7 @@ class ShareGroupHeartbeatRequestTest(cluster: ClusterInstance) {
val allPartitionsMetadata = waitForAllPartitionsMetadata(brokersToValidate, topic, totalPartitionCount)
(0 until totalPartitionCount - 1).foreach(i => {
allPartitionsMetadata.get(new TopicPartition(topic, i)).foreach { partitionMetadata =>
- assertEquals(totalPartitionCount, partitionMetadata.replicas.size)
+ assertEquals(totalPartitionCount, partitionMetadata.isr.size)
}
})
}
diff --git a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
index d3bed2905f6..e32f039b9b5 100644
--- a/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
+++ b/core/src/test/scala/unit/kafka/server/epoch/LeaderEpochIntegrationTest.scala
@@ -245,7 +245,7 @@ class LeaderEpochIntegrationTest extends QuorumTestHarness with Logging {
private def waitForEpochChangeTo(topic: String, partition: Int, epoch: Int): Unit = {
TestUtils.waitUntilTrue(() => {
- brokers(0).metadataCache.getPartitionInfo(topic, partition).exists(_.leaderEpoch == epoch)
+ brokers(0).metadataCache.getLeaderAndIsr(topic, partition).exists(_.leaderEpoch == epoch)
}, "Epoch didn't change")
}
diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
index 580816b32e7..7540e129477 100755
--- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala
+++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala
@@ -36,7 +36,6 @@ import org.apache.kafka.common.errors.{OperationNotAttemptedException, TopicExis
import org.apache.kafka.common.header.Header
import org.apache.kafka.common.internals.Topic
import org.apache.kafka.common.memory.MemoryPool
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.network.{ClientInformation, ConnectionMode, ListenerName}
import org.apache.kafka.common.protocol.{ApiKeys, Errors}
@@ -823,7 +822,7 @@ object TestUtils extends Logging {
def waitForAllPartitionsMetadata[B <: KafkaBroker](
brokers: Seq[B],
topic: String,
- expectedNumPartitions: Int): Map[TopicPartition, UpdateMetadataPartitionState] = {
+ expectedNumPartitions: Int): Map[TopicPartition, LeaderAndIsr] = {
waitUntilTrue(
() => brokers.forall { broker =>
if (expectedNumPartitions == 0) {
@@ -836,7 +835,7 @@ object TestUtils extends Logging {
// since the metadata is propagated, we should get the same metadata from each server
(0 until expectedNumPartitions).map { i =>
- new TopicPartition(topic, i) -> brokers.head.metadataCache.getPartitionInfo(topic, i).getOrElse(
+ new TopicPartition(topic, i) -> brokers.head.metadataCache.getLeaderAndIsr(topic, i).getOrElse(
throw new IllegalStateException(s"Cannot get topic: $topic, partition: $i in server metadata cache"))
}.toMap
}
@@ -853,10 +852,10 @@ object TestUtils extends Logging {
*/
def waitForPartitionMetadata[B <: KafkaBroker](
brokers: Seq[B], topic: String, partition: Int,
- timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): UpdateMetadataPartitionState = {
+ timeout: Long = JTestUtils.DEFAULT_MAX_WAIT_MS): LeaderAndIsr = {
waitUntilTrue(
() => brokers.forall { broker =>
- broker.metadataCache.getPartitionInfo(topic, partition) match {
+ broker.metadataCache.getLeaderAndIsr(topic, partition) match {
case Some(partitionState) => FetchRequest.isValidBrokerId(partitionState.leader)
case _ => false
}
@@ -864,7 +863,7 @@ object TestUtils extends Logging {
"Partition [%s,%d] metadata not propagated after %d ms".format(topic, partition, timeout),
waitTimeMs = timeout)
- brokers.head.metadataCache.getPartitionInfo(topic, partition).getOrElse(
+ brokers.head.metadataCache.getLeaderAndIsr(topic, partition).getOrElse(
throw new IllegalStateException(s"Cannot get topic: $topic, partition: $partition in server metadata cache"))
}
diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
index 7aa9d23d73b..9448c3cf35c 100644
--- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java
@@ -62,6 +62,8 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
String requestSchema() {
if (requestSpec == null) {
return "null";
+ } else if (!requestSpec.hasValidVersion()) {
+ return "new Schema[0]";
} else {
return String.format("%sData.SCHEMAS", requestSpec.name());
}
@@ -70,6 +72,8 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
String responseSchema() {
if (responseSpec == null) {
return "null";
+ } else if (!requestSpec.hasValidVersion()) {
+ return "new Schema[0]";
} else {
return String.format("%sData.SCHEMAS", responseSpec.name());
}
@@ -301,12 +305,15 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
buffer.printf("switch (apiKey) {%n");
buffer.incrementIndent();
for (Map.Entry<Short, ApiData> entry : apis.entrySet()) {
- buffer.printf("case %d:%n", entry.getKey());
- buffer.incrementIndent();
- buffer.printf("return new %s%sData();%n",
- entry.getValue().name(),
- MessageGenerator.capitalizeFirst(type));
- buffer.decrementIndent();
+ MessageSpec spec = messageSpec(type, entry.getKey(), entry.getValue());
+ if (spec.hasValidVersion()) {
+ buffer.printf("case %d:%n", entry.getKey());
+ buffer.incrementIndent();
+ buffer.printf("return new %s%sData();%n",
+ entry.getValue().name(),
+ MessageGenerator.capitalizeFirst(type));
+ buffer.decrementIndent();
+ }
}
buffer.printf("default:%n");
buffer.incrementIndent();
@@ -346,6 +353,11 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
short apiKey = entry.getKey();
ApiData apiData = entry.getValue();
String name = apiData.name();
+
+ MessageSpec spec = messageSpec(type, apiKey, entry.getValue());
+ if (!spec.hasValidVersion())
+ continue;
+
buffer.printf("case %d: // %s%n", apiKey, MessageGenerator.capitalizeFirst(name));
buffer.incrementIndent();
if (type.equals("response") && apiKey == 18) {
@@ -355,28 +367,6 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
buffer.decrementIndent();
continue;
}
- if (type.equals("request") && apiKey == 7) {
- buffer.printf("// Version 0 of ControlledShutdownRequest has a non-standard request header%n");
- buffer.printf("// which does not include clientId. Version 1 of ControlledShutdownRequest%n");
- buffer.printf("// and later use the standard request header.%n");
- buffer.printf("if (_version == 0) {%n");
- buffer.incrementIndent();
- buffer.printf("return (short) 0;%n");
- buffer.decrementIndent();
- buffer.printf("}%n");
- }
- ApiData data = entry.getValue();
- MessageSpec spec;
- if (type.equals("request")) {
- spec = data.requestSpec;
- } else if (type.equals("response")) {
- spec = data.responseSpec;
- } else {
- throw new RuntimeException("Invalid type " + type + " for generateHeaderVersion");
- }
- if (spec == null) {
- throw new RuntimeException("failed to find " + type + " for API key " + apiKey);
- }
VersionConditional.forVersions(spec.flexibleVersions(),
spec.validVersions()).
ifMember(__ -> {
@@ -407,6 +397,20 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator {
buffer.printf("}%n");
}
+ private static MessageSpec messageSpec(String type, short apiKey, ApiData apiData) {
+ MessageSpec spec;
+ if (type.equals("request")) {
+ spec = apiData.requestSpec;
+ } else if (type.equals("response")) {
+ spec = apiData.responseSpec;
+ } else {
+ throw new RuntimeException("Invalid type " + type);
+ }
+ if (spec == null)
+ throw new RuntimeException("failed to find " + type + " for API key " + apiKey);
+ return spec;
+ }
+
private void generateListenerTypesEnum() {
buffer.printf("public enum ListenerType {%n");
buffer.incrementIndent();
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
index 4d7903cd6af..5cbb061fac3 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java
@@ -29,6 +29,7 @@ import net.sourceforge.argparse4j.inf.ArgumentParser;
import net.sourceforge.argparse4j.inf.Namespace;
import java.io.BufferedWriter;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
@@ -39,6 +40,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Set;
import static net.sourceforge.argparse4j.impl.Arguments.store;
@@ -227,25 +229,14 @@ public final class MessageGenerator {
Files.createDirectories(Paths.get(outputDir));
int numProcessed = 0;
- List<TypeClassGenerator> typeClassGenerators =
- createTypeClassGenerators(packageName, typeClassGeneratorTypes);
- HashSet<String> outputFileNames = new HashSet<>();
- try (DirectoryStream<Path> directoryStream = Files
- .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+ List<TypeClassGenerator> typeClassGenerators = createTypeClassGenerators(packageName, typeClassGeneratorTypes);
+ Set<String> outputFileNames = new HashSet<>();
+ try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
for (Path inputPath : directoryStream) {
try {
- MessageSpec spec = JSON_SERDE.
- readValue(inputPath.toFile(), MessageSpec.class);
- List<MessageClassGenerator> generators =
- createMessageClassGenerators(packageName, messageClassGeneratorTypes);
- for (MessageClassGenerator generator : generators) {
- String name = generator.outputName(spec) + JAVA_SUFFIX;
- outputFileNames.add(name);
- Path outputPath = Paths.get(outputDir, name);
- try (BufferedWriter writer = Files.newBufferedWriter(outputPath, StandardCharsets.UTF_8)) {
- generator.generateAndWrite(spec, writer);
- }
- }
+ MessageSpec spec = JSON_SERDE.readValue(inputPath.toFile(), MessageSpec.class);
+ outputFileNames.addAll(
+ generateAndWriteMessageClasses(spec, packageName, outputDir, messageClassGeneratorTypes));
numProcessed++;
typeClassGenerators.forEach(generator -> generator.registerMessageType(spec));
} catch (Exception e) {
@@ -255,13 +246,10 @@ public final class MessageGenerator {
}
for (TypeClassGenerator typeClassGenerator : typeClassGenerators) {
outputFileNames.add(typeClassGenerator.outputName());
- Path factoryOutputPath = Paths.get(outputDir, typeClassGenerator.outputName());
- try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath, StandardCharsets.UTF_8)) {
- typeClassGenerator.generateAndWrite(writer);
- }
+ generateAndWriteTypeClasses(outputDir, typeClassGenerator);
}
- try (DirectoryStream<Path> directoryStream = Files.
- newDirectoryStream(Paths.get(outputDir))) {
+
+ try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(Paths.get(outputDir))) {
for (Path outputPath : directoryStream) {
Path fileName = outputPath.getFileName();
if (fileName != null) {
@@ -274,6 +262,37 @@ public final class MessageGenerator {
System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);
}
+ /**
+ * Generate and write message classes.
+ * @return output file names.
+ */
+ static Set<String> generateAndWriteMessageClasses(MessageSpec spec,
+ String packageName,
+ String outputDir,
+ List<String> messageClassGeneratorTypes) throws Exception {
+ var outputFileNames = new HashSet<String>();
+ // Only generate `Data` classes for apis that have valid version(s)
+ if (spec.hasValidVersion()) {
+ List<MessageClassGenerator> generators = createMessageClassGenerators(packageName, messageClassGeneratorTypes);
+ for (MessageClassGenerator generator : generators) {
+ String name = generator.outputName(spec) + JAVA_SUFFIX;
+ outputFileNames.add(name);
+ Path outputPath = Paths.get(outputDir, name);
+ try (BufferedWriter writer = Files.newBufferedWriter(outputPath, StandardCharsets.UTF_8)) {
+ generator.generateAndWrite(spec, writer);
+ }
+ }
+ }
+ return outputFileNames;
+ }
+
+ static void generateAndWriteTypeClasses(String outputDir, TypeClassGenerator typeClassGenerator) throws IOException {
+ Path factoryOutputPath = Paths.get(outputDir, typeClassGenerator.outputName());
+ try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath, StandardCharsets.UTF_8)) {
+ typeClassGenerator.generateAndWrite(writer);
+ }
+ }
+
public static String capitalizeFirst(String string) {
if (string.isEmpty()) {
return string;
diff --git a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
index 66a720f2ffd..f6620fb6ec1 100644
--- a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
+++ b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java
@@ -20,7 +20,6 @@ package org.apache.kafka.message;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -57,30 +56,39 @@ public final class MessageSpec {
this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey);
this.type = Objects.requireNonNull(type);
this.commonStructs = commonStructs == null ? Collections.emptyList() :
- Collections.unmodifiableList(new ArrayList<>(commonStructs));
- if (flexibleVersions == null) {
- throw new RuntimeException("You must specify a value for flexibleVersions. " +
- "Please use 0+ for all new messages.");
+ List.copyOf(commonStructs);
+ // If the struct has no valid versions (the typical use case is to completely remove support for
+ // an existing protocol api while ensuring the api key id is not reused), we configure the spec
+ // to effectively be empty
+ if (struct.versions().empty()) {
+ this.flexibleVersions = Versions.NONE;
+ this.listeners = Collections.emptyList();
+ this.latestVersionUnstable = false;
+ } else {
+ if (flexibleVersions == null) {
+ throw new RuntimeException("You must specify a value for flexibleVersions. " +
+ "Please use 0+ for all new messages.");
+ }
+ this.flexibleVersions = Versions.parse(flexibleVersions, Versions.NONE);
+ if ((!this.flexibleVersions().empty()) &&
+ (this.flexibleVersions.highest() < Short.MAX_VALUE)) {
+ throw new RuntimeException("Field " + name + " specifies flexibleVersions " +
+ this.flexibleVersions + ", which is not open-ended. flexibleVersions must " +
+ "be either none, or an open-ended range (that ends with a plus sign).");
+ }
+
+ if (listeners != null && !listeners.isEmpty() && type != MessageSpecType.REQUEST) {
+ throw new RuntimeException("The `requestScope` property is only valid for " +
+ "messages with type `request`");
+ }
+ this.listeners = listeners;
+
+ if (latestVersionUnstable && type != MessageSpecType.REQUEST) {
+ throw new RuntimeException("The `latestVersionUnstable` property is only valid for " +
+ "messages with type `request`");
+ }
+ this.latestVersionUnstable = latestVersionUnstable;
}
- this.flexibleVersions = Versions.parse(flexibleVersions, Versions.NONE);
- if ((!this.flexibleVersions().empty()) &&
- (this.flexibleVersions.highest() < Short.MAX_VALUE)) {
- throw new RuntimeException("Field " + name + " specifies flexibleVersions " +
- this.flexibleVersions + ", which is not open-ended. flexibleVersions must " +
- "be either none, or an open-ended range (that ends with a plus sign).");
- }
-
- if (listeners != null && !listeners.isEmpty() && type != MessageSpecType.REQUEST) {
- throw new RuntimeException("The `requestScope` property is only valid for " +
- "messages with type `request`");
- }
- this.listeners = listeners;
-
- if (latestVersionUnstable && type != MessageSpecType.REQUEST) {
- throw new RuntimeException("The `latestVersionUnstable` property is only valid for " +
- "messages with type `request`");
- }
- this.latestVersionUnstable = latestVersionUnstable;
}
public StructSpec struct() {
@@ -92,6 +100,10 @@ public final class MessageSpec {
return struct.name();
}
+ public boolean hasValidVersion() {
+ return !struct.versions().empty();
+ }
+
public Versions validVersions() {
return struct.versions();
}
diff --git a/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
index fd25d5e774b..7fa736ee9a4 100644
--- a/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java
@@ -19,6 +19,14 @@ package org.apache.kafka.message;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.BufferedWriter;
+import java.io.StringWriter;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -69,4 +77,70 @@ public class MessageGeneratorTest {
assertEquals(MessageGenerator.UNSIGNED_SHORT_MAX, 0xFFFF);
assertEquals(MessageGenerator.UNSIGNED_INT_MAX, 0xFFFFFFFFL);
}
+
+ @Test
+ public void testGenerateAndWriteMessageClasses(@TempDir Path tempDir) throws Exception {
+ var generatorTypes = List.of("MessageDataGenerator", "JsonConverterGenerator");
+
+ MessageSpec testRequestSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", List.of(
+ "{",
+ " \"apiKey\": 0,",
+ " \"type\": \"request\",",
+ " \"name\": \"FooBarRequest\",",
+ " \"validVersions\": \"none\"",
+ "}")), MessageSpec.class);
+ MessageSpec testResponseSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", List.of(
+ "{",
+ " \"apiKey\": 0,",
+ " \"type\": \"response\",",
+ " \"name\": \"FooBarRespose\",",
+ " \"validVersions\": \"none\"",
+ "}")), MessageSpec.class);
+
+ var outputFiles = MessageGenerator.generateAndWriteMessageClasses(testRequestSpec, "kafka",
+ tempDir.toAbsolutePath().toString(), generatorTypes);
+ assertEquals(Set.of(), outputFiles);
+ outputFiles = MessageGenerator.generateAndWriteMessageClasses(testResponseSpec, "kafka",
+ tempDir.toAbsolutePath().toString(), generatorTypes);
+ assertEquals(Set.of(), outputFiles);
+ var typeGenerator = new ApiMessageTypeGenerator("kafka");
+ typeGenerator.registerMessageType(testRequestSpec);
+ typeGenerator.registerMessageType(testResponseSpec);
+ typeGenerator.generateAndWrite(new BufferedWriter(new StringWriter()));
+
+ testRequestSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+ "{",
+ " \"apiKey\": 0,",
+ " \"type\": \"request\",",
+ " \"name\": \"FooBarRequest\",",
+ " \"validVersions\": \"0-2\",",
+ " \"flexibleVersions\": \"none\",",
+ " \"fields\": [",
+ " { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\" }",
+ " ]",
+ "}")), MessageSpec.class);
+ testResponseSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", Arrays.asList(
+ "{",
+ " \"apiKey\": 0,",
+ " \"type\": \"response\",",
+ " \"name\": \"FooBarResponse\",",
+ " \"validVersions\": \"0-2\",",
+ " \"flexibleVersions\": \"none\",",
+ " \"fields\": [",
+ " { \"name\": \"field1\", \"type\": \"int32\", \"versions\": \"0+\" }",
+ " ]",
+ "}")), MessageSpec.class);
+
+ outputFiles = MessageGenerator.generateAndWriteMessageClasses(testRequestSpec, "kafka",
+ tempDir.toAbsolutePath().toString(), generatorTypes);
+ assertEquals(Set.of("FooBarRequestDataJsonConverter.java", "FooBarRequestData.java"), outputFiles);
+ outputFiles = MessageGenerator.generateAndWriteMessageClasses(testResponseSpec, "kafka",
+ tempDir.toAbsolutePath().toString(), generatorTypes);
+ assertEquals(Set.of("FooBarResponseDataJsonConverter.java", "FooBarResponseData.java"), outputFiles);
+ typeGenerator = new ApiMessageTypeGenerator("kafka");
+ typeGenerator.registerMessageType(testRequestSpec);
+ typeGenerator.registerMessageType(testResponseSpec);
+ typeGenerator.generateAndWrite(new BufferedWriter(new StringWriter()));
+ }
+
}
diff --git a/generator/src/test/java/org/apache/kafka/message/StructRegistryTest.java b/generator/src/test/java/org/apache/kafka/message/StructRegistryTest.java
index 5e27f9ad1b9..fa1136c68f6 100644
--- a/generator/src/test/java/org/apache/kafka/message/StructRegistryTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/StructRegistryTest.java
@@ -22,6 +22,7 @@ import org.junit.jupiter.api.Timeout;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -150,4 +151,21 @@ public class StructRegistryTest {
assertEquals(structRegistry.findStruct(field2).name(), "TestInlineStruct");
assertFalse(structRegistry.isStructArrayWithKeys(field2));
}
+
+ @Test
+ public void testValidVersionsIsNone() throws Exception {
+ MessageSpec testMessageSpec = MessageGenerator.JSON_SERDE.readValue(String.join("", List.of(
+ "{",
+ " \"type\": \"request\",",
+ " \"name\": \"FooBar\",",
+ " \"validVersions\": \"none\"",
+ "}")), MessageSpec.class);
+ StructRegistry structRegistry = new StructRegistry();
+ structRegistry.register(testMessageSpec);
+
+ assertFalse(testMessageSpec.hasValidVersion());
+ assertEquals(List.of(), testMessageSpec.fields());
+ assertFalse(structRegistry.structs().hasNext());
+ assertFalse(structRegistry.commonStructs().hasNext());
+ }
}
diff --git a/generator/src/test/java/org/apache/kafka/message/VersionsTest.java b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
index a61a1390aae..2bff70d064b 100644
--- a/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
+++ b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java
@@ -40,6 +40,7 @@ public class VersionsTest {
@Test
public void testVersionsParse() {
+ assertEquals(Versions.NONE, Versions.parse(" none ", null));
assertEquals(Versions.NONE, Versions.parse(null, Versions.NONE));
assertEquals(Versions.ALL, Versions.parse(" ", Versions.ALL));
assertEquals(Versions.ALL, Versions.parse("", Versions.ALL));
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java
index ae0a582d0a0..3b0afed5000 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java
@@ -41,7 +41,6 @@ import org.apache.kafka.common.TopicIdPartition;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.message.FetchResponseData;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData;
import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition;
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset;
import org.apache.kafka.common.metrics.Metrics;
@@ -51,6 +50,7 @@ import org.apache.kafka.common.record.BaseRecords;
import org.apache.kafka.common.record.RecordsSend;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.FetchResponse;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
@@ -168,7 +168,7 @@ public class ReplicaFetcherThreadBenchmark {
TopicPartition tp = new TopicPartition("topic", i);
List<Integer> replicas = Arrays.asList(0, 1, 2);
- LeaderAndIsrRequestData.LeaderAndIsrPartitionState partitionState = new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
+ LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(0)
.setLeaderEpoch(0)
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java
index a46a4d94bbe..d58b35942fe 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/KRaftMetadataRequestBenchmark.java
@@ -40,7 +40,6 @@ import kafka.server.share.SharePartitionManager;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.memory.MemoryPool;
import org.apache.kafka.common.message.ApiMessageType;
-import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint;
import org.apache.kafka.common.metadata.PartitionRecord;
import org.apache.kafka.common.metadata.RegisterBrokerRecord;
import org.apache.kafka.common.metadata.TopicRecord;
@@ -141,12 +140,7 @@ public class KRaftMetadataRequestBenchmark {
MetadataDelta buildupMetadataDelta = new MetadataDelta(MetadataImage.EMPTY);
IntStream.range(0, 5).forEach(brokerId -> {
RegisterBrokerRecord.BrokerEndpointCollection endpoints = new RegisterBrokerRecord.BrokerEndpointCollection();
- endpoints(brokerId).forEach(endpoint ->
- endpoints.add(new RegisterBrokerRecord.BrokerEndpoint().
- setHost(endpoint.host()).
- setPort(endpoint.port()).
- setName(endpoint.listener()).
- setSecurityProtocol(endpoint.securityProtocol())));
+ endpoints(brokerId).forEach(endpoint -> endpoints.add(endpoint));
buildupMetadataDelta.replay(new RegisterBrokerRecord().
setBrokerId(brokerId).
setBrokerEpoch(100L).
@@ -172,13 +166,13 @@ public class KRaftMetadataRequestBenchmark {
metadataCache.setImage(buildupMetadataDelta.apply(MetadataProvenance.EMPTY));
}
- private List<UpdateMetadataEndpoint> endpoints(final int brokerId) {
+ private List<RegisterBrokerRecord.BrokerEndpoint> endpoints(final int brokerId) {
return Collections.singletonList(
- new UpdateMetadataEndpoint()
- .setHost("host_" + brokerId)
- .setPort(9092)
- .setSecurityProtocol(SecurityProtocol.PLAINTEXT.id)
- .setListener(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT).value()));
+ new RegisterBrokerRecord.BrokerEndpoint().
+ setHost("host_" + brokerId).
+ setPort(9092).
+ setName(ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT).value()).
+ setSecurityProtocol(SecurityProtocol.PLAINTEXT.id));
}
private KafkaApis createKafkaApis() {
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/PartitionMakeFollowerBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/PartitionMakeFollowerBenchmark.java
index dd636829910..16ef047ca27 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/PartitionMakeFollowerBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/PartitionMakeFollowerBenchmark.java
@@ -29,9 +29,9 @@ import kafka.server.metadata.MockConfigRepository;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.compress.Compression;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData;
import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.SimpleRecord;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.server.common.MetadataVersion;
@@ -152,7 +152,7 @@ public class PartitionMakeFollowerBenchmark {
@Benchmark
public boolean testMakeFollower() {
- LeaderAndIsrRequestData.LeaderAndIsrPartitionState partitionState = new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
+ LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(0)
.setLeaderEpoch(0)
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
index 13b3c0a1fe4..0fb7206301c 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/partition/UpdateFollowerFetchStateBenchmark.java
@@ -29,7 +29,7 @@ import kafka.server.metadata.MockConfigRepository;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.server.common.MetadataVersion;
import org.apache.kafka.server.util.KafkaScheduler;
@@ -117,7 +117,7 @@ public class UpdateFollowerFetchStateBenchmark {
replicas.add(0);
replicas.add(1);
replicas.add(2);
- LeaderAndIsrPartitionState partitionState = new LeaderAndIsrPartitionState()
+ LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(0)
.setLeaderEpoch(0)
diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/PartitionCreationBench.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/PartitionCreationBench.java
index 4a50e05433c..1ced9ba50b0 100644
--- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/PartitionCreationBench.java
+++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/server/PartitionCreationBench.java
@@ -31,8 +31,8 @@ import kafka.utils.TestUtils;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData;
import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.server.common.MetadataVersion;
@@ -200,7 +200,7 @@ public class PartitionCreationBench {
inSync.add(1);
inSync.add(2);
- LeaderAndIsrRequestData.LeaderAndIsrPartitionState partitionState = new LeaderAndIsrRequestData.LeaderAndIsrPartitionState()
+ LeaderAndIsrRequest.PartitionState partitionState = new LeaderAndIsrRequest.PartitionState()
.setControllerEpoch(0)
.setLeader(0)
.setLeaderEpoch(0)
diff --git a/metadata/src/main/java/org/apache/kafka/metadata/PartitionRegistration.java b/metadata/src/main/java/org/apache/kafka/metadata/PartitionRegistration.java
index 59c3763ca72..b50917e3275 100644
--- a/metadata/src/main/java/org/apache/kafka/metadata/PartitionRegistration.java
+++ b/metadata/src/main/java/org/apache/kafka/metadata/PartitionRegistration.java
@@ -21,9 +21,9 @@ import org.apache.kafka.common.DirectoryId;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.errors.InvalidReplicaDirectoriesException;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
import org.apache.kafka.common.metadata.PartitionChangeRecord;
import org.apache.kafka.common.metadata.PartitionRecord;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.image.writer.ImageWriterOptions;
import org.apache.kafka.server.common.ApiMessageAndVersion;
@@ -406,9 +406,9 @@ public class PartitionRegistration {
return new ApiMessageAndVersion(record, options.metadataVersion().partitionRecordVersion());
}
- public LeaderAndIsrPartitionState toLeaderAndIsrPartitionState(TopicPartition tp,
- boolean isNew) {
- return new LeaderAndIsrPartitionState().
+ public LeaderAndIsrRequest.PartitionState toLeaderAndIsrPartitionState(TopicPartition tp,
+ boolean isNew) {
+ return new LeaderAndIsrRequest.PartitionState().
setTopicName(tp.topic()).
setPartitionIndex(tp.partition()).
setControllerEpoch(-1).
diff --git a/metadata/src/test/java/org/apache/kafka/metadata/PartitionRegistrationTest.java b/metadata/src/test/java/org/apache/kafka/metadata/PartitionRegistrationTest.java
index b6a5cd20a16..c7d1767889b 100644
--- a/metadata/src/test/java/org/apache/kafka/metadata/PartitionRegistrationTest.java
+++ b/metadata/src/test/java/org/apache/kafka/metadata/PartitionRegistrationTest.java
@@ -20,9 +20,9 @@ package org.apache.kafka.metadata;
import org.apache.kafka.common.DirectoryId;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
-import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState;
import org.apache.kafka.common.metadata.PartitionChangeRecord;
import org.apache.kafka.common.metadata.PartitionRecord;
+import org.apache.kafka.common.requests.LeaderAndIsrRequest;
import org.apache.kafka.image.writer.ImageWriterOptions;
import org.apache.kafka.image.writer.UnwritableMetadataException;
import org.apache.kafka.server.common.ApiMessageAndVersion;
@@ -116,7 +116,7 @@ public class PartitionRegistrationTest {
Uuid.fromString("bAAlGAz1TN2doZjtWlvhRQ")
}).
setIsr(new int[]{2, 3, 4}).setLeader(2).setLeaderRecoveryState(LeaderRecoveryState.RECOVERED).setLeaderEpoch(234).setPartitionEpoch(567).build();
- assertEquals(new LeaderAndIsrPartitionState().
+ assertEquals(new LeaderAndIsrRequest.PartitionState().
setTopicName("foo").
setPartitionIndex(1).
setControllerEpoch(-1).
@@ -129,7 +129,7 @@ public class PartitionRegistrationTest {
setRemovingReplicas(Collections.emptyList()).
setIsNew(true).toString(),
a.toLeaderAndIsrPartitionState(new TopicPartition("foo", 1), true).toString());
- assertEquals(new LeaderAndIsrPartitionState().
+ assertEquals(new LeaderAndIsrRequest.PartitionState().
setTopicName("bar").
setPartitionIndex(0).
setControllerEpoch(-1).
diff --git a/server/src/main/java/org/apache/kafka/network/RequestConvertToJson.java b/server/src/main/java/org/apache/kafka/network/RequestConvertToJson.java
index 63e188dfbaf..2abea1d62dd 100644
--- a/server/src/main/java/org/apache/kafka/network/RequestConvertToJson.java
+++ b/server/src/main/java/org/apache/kafka/network/RequestConvertToJson.java
@@ -50,8 +50,6 @@ import org.apache.kafka.common.message.ConsumerGroupDescribeRequestDataJsonConve
import org.apache.kafka.common.message.ConsumerGroupDescribeResponseDataJsonConverter;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestDataJsonConverter;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseDataJsonConverter;
-import org.apache.kafka.common.message.ControlledShutdownRequestDataJsonConverter;
-import org.apache.kafka.common.message.ControlledShutdownResponseDataJsonConverter;
import org.apache.kafka.common.message.ControllerRegistrationRequestDataJsonConverter;
import org.apache.kafka.common.message.ControllerRegistrationResponseDataJsonConverter;
import org.apache.kafka.common.message.CreateAclsRequestDataJsonConverter;
@@ -124,8 +122,6 @@ import org.apache.kafka.common.message.InitializeShareGroupStateRequestDataJsonC
import org.apache.kafka.common.message.InitializeShareGroupStateResponseDataJsonConverter;
import org.apache.kafka.common.message.JoinGroupRequestDataJsonConverter;
import org.apache.kafka.common.message.JoinGroupResponseDataJsonConverter;
-import org.apache.kafka.common.message.LeaderAndIsrRequestDataJsonConverter;
-import org.apache.kafka.common.message.LeaderAndIsrResponseDataJsonConverter;
import org.apache.kafka.common.message.LeaveGroupRequestDataJsonConverter;
import org.apache.kafka.common.message.LeaveGroupResponseDataJsonConverter;
import org.apache.kafka.common.message.ListClientMetricsResourcesRequestDataJsonConverter;
@@ -173,8 +169,6 @@ import org.apache.kafka.common.message.ShareGroupDescribeRequestDataJsonConverte
import org.apache.kafka.common.message.ShareGroupDescribeResponseDataJsonConverter;
import org.apache.kafka.common.message.ShareGroupHeartbeatRequestDataJsonConverter;
import org.apache.kafka.common.message.ShareGroupHeartbeatResponseDataJsonConverter;
-import org.apache.kafka.common.message.StopReplicaRequestDataJsonConverter;
-import org.apache.kafka.common.message.StopReplicaResponseDataJsonConverter;
import org.apache.kafka.common.message.SyncGroupRequestDataJsonConverter;
import org.apache.kafka.common.message.SyncGroupResponseDataJsonConverter;
import org.apache.kafka.common.message.TxnOffsetCommitRequestDataJsonConverter;
@@ -183,8 +177,6 @@ import org.apache.kafka.common.message.UnregisterBrokerRequestDataJsonConverter;
import org.apache.kafka.common.message.UnregisterBrokerResponseDataJsonConverter;
import org.apache.kafka.common.message.UpdateFeaturesRequestDataJsonConverter;
import org.apache.kafka.common.message.UpdateFeaturesResponseDataJsonConverter;
-import org.apache.kafka.common.message.UpdateMetadataRequestDataJsonConverter;
-import org.apache.kafka.common.message.UpdateMetadataResponseDataJsonConverter;
import org.apache.kafka.common.message.UpdateRaftVoterRequestDataJsonConverter;
import org.apache.kafka.common.message.UpdateRaftVoterResponseDataJsonConverter;
import org.apache.kafka.common.message.VoteRequestDataJsonConverter;
@@ -230,8 +222,6 @@ import org.apache.kafka.common.requests.ConsumerGroupDescribeRequest;
import org.apache.kafka.common.requests.ConsumerGroupDescribeResponse;
import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest;
import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse;
-import org.apache.kafka.common.requests.ControlledShutdownRequest;
-import org.apache.kafka.common.requests.ControlledShutdownResponse;
import org.apache.kafka.common.requests.ControllerRegistrationRequest;
import org.apache.kafka.common.requests.ControllerRegistrationResponse;
import org.apache.kafka.common.requests.CreateAclsRequest;
@@ -304,8 +294,6 @@ import org.apache.kafka.common.requests.InitializeShareGroupStateRequest;
import org.apache.kafka.common.requests.InitializeShareGroupStateResponse;
import org.apache.kafka.common.requests.JoinGroupRequest;
import org.apache.kafka.common.requests.JoinGroupResponse;
-import org.apache.kafka.common.requests.LeaderAndIsrRequest;
-import org.apache.kafka.common.requests.LeaderAndIsrResponse;
import org.apache.kafka.common.requests.LeaveGroupRequest;
import org.apache.kafka.common.requests.LeaveGroupResponse;
import org.apache.kafka.common.requests.ListClientMetricsResourcesRequest;
@@ -354,8 +342,6 @@ import org.apache.kafka.common.requests.ShareGroupDescribeRequest;
import org.apache.kafka.common.requests.ShareGroupDescribeResponse;
import org.apache.kafka.common.requests.ShareGroupHeartbeatRequest;
import org.apache.kafka.common.requests.ShareGroupHeartbeatResponse;
-import org.apache.kafka.common.requests.StopReplicaRequest;
-import org.apache.kafka.common.requests.StopReplicaResponse;
import org.apache.kafka.common.requests.SyncGroupRequest;
import org.apache.kafka.common.requests.SyncGroupResponse;
import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
@@ -364,8 +350,6 @@ import org.apache.kafka.common.requests.UnregisterBrokerRequest;
import org.apache.kafka.common.requests.UnregisterBrokerResponse;
import org.apache.kafka.common.requests.UpdateFeaturesRequest;
import org.apache.kafka.common.requests.UpdateFeaturesResponse;
-import org.apache.kafka.common.requests.UpdateMetadataRequest;
-import org.apache.kafka.common.requests.UpdateMetadataResponse;
import org.apache.kafka.common.requests.UpdateRaftVoterRequest;
import org.apache.kafka.common.requests.UpdateRaftVoterResponse;
import org.apache.kafka.common.requests.VoteRequest;
@@ -421,8 +405,6 @@ public class RequestConvertToJson {
return ConsumerGroupDescribeRequestDataJsonConverter.write(((ConsumerGroupDescribeRequest) request).data(), request.version());
case CONSUMER_GROUP_HEARTBEAT:
return ConsumerGroupHeartbeatRequestDataJsonConverter.write(((ConsumerGroupHeartbeatRequest) request).data(), request.version());
- case CONTROLLED_SHUTDOWN:
- return ControlledShutdownRequestDataJsonConverter.write(((ControlledShutdownRequest) request).data(), request.version());
case CONTROLLER_REGISTRATION:
return ControllerRegistrationRequestDataJsonConverter.write(((ControllerRegistrationRequest) request).data(), request.version());
case CREATE_ACLS:
@@ -495,8 +477,6 @@ public class RequestConvertToJson {
return InitProducerIdRequestDataJsonConverter.write(((InitProducerIdRequest) request).data(), request.version());
case JOIN_GROUP:
return JoinGroupRequestDataJsonConverter.write(((JoinGroupRequest) request).data(), request.version());
- case LEADER_AND_ISR:
- return LeaderAndIsrRequestDataJsonConverter.write(((LeaderAndIsrRequest) request).data(), request.version());
case LEAVE_GROUP:
return LeaveGroupRequestDataJsonConverter.write(((LeaveGroupRequest) request).data(), request.version());
case LIST_CLIENT_METRICS_RESOURCES:
@@ -541,8 +521,6 @@ public class RequestConvertToJson {
return ShareGroupDescribeRequestDataJsonConverter.write(((ShareGroupDescribeRequest) request).data(), request.version());
case SHARE_GROUP_HEARTBEAT:
return ShareGroupHeartbeatRequestDataJsonConverter.write(((ShareGroupHeartbeatRequest) request).data(), request.version());
- case STOP_REPLICA:
- return StopReplicaRequestDataJsonConverter.write(((StopReplicaRequest) request).data(), request.version());
case SYNC_GROUP:
return SyncGroupRequestDataJsonConverter.write(((SyncGroupRequest) request).data(), request.version());
case TXN_OFFSET_COMMIT:
@@ -551,8 +529,6 @@ public class RequestConvertToJson {
return UnregisterBrokerRequestDataJsonConverter.write(((UnregisterBrokerRequest) request).data(), request.version());
case UPDATE_FEATURES:
return UpdateFeaturesRequestDataJsonConverter.write(((UpdateFeaturesRequest) request).data(), request.version());
- case UPDATE_METADATA:
- return UpdateMetadataRequestDataJsonConverter.write(((UpdateMetadataRequest) request).data(), request.version());
case VOTE:
return VoteRequestDataJsonConverter.write(((VoteRequest) request).data(), request.version());
case WRITE_SHARE_GROUP_STATE:
@@ -605,8 +581,6 @@ public class RequestConvertToJson {
return ConsumerGroupDescribeResponseDataJsonConverter.write(((ConsumerGroupDescribeResponse) response).data(), version);
case CONSUMER_GROUP_HEARTBEAT:
return ConsumerGroupHeartbeatResponseDataJsonConverter.write(((ConsumerGroupHeartbeatResponse) response).data(), version);
- case CONTROLLED_SHUTDOWN:
- return ControlledShutdownResponseDataJsonConverter.write(((ControlledShutdownResponse) response).data(), version);
case CONTROLLER_REGISTRATION:
return ControllerRegistrationResponseDataJsonConverter.write(((ControllerRegistrationResponse) response).data(), version);
case CREATE_ACLS:
@@ -679,8 +653,6 @@ public class RequestConvertToJson {
return InitProducerIdResponseDataJsonConverter.write(((InitProducerIdResponse) response).data(), version);
case JOIN_GROUP:
return JoinGroupResponseDataJsonConverter.write(((JoinGroupResponse) response).data(), version);
- case LEADER_AND_ISR:
- return LeaderAndIsrResponseDataJsonConverter.write(((LeaderAndIsrResponse) response).data(), version);
case LEAVE_GROUP:
return LeaveGroupResponseDataJsonConverter.write(((LeaveGroupResponse) response).data(), version);
case LIST_CLIENT_METRICS_RESOURCES:
@@ -725,8 +697,6 @@ public class RequestConvertToJson {
return ShareGroupDescribeResponseDataJsonConverter.write(((ShareGroupDescribeResponse) response).data(), version);
case SHARE_GROUP_HEARTBEAT:
return ShareGroupHeartbeatResponseDataJsonConverter.write(((ShareGroupHeartbeatResponse) response).data(), version);
- case STOP_REPLICA:
- return StopReplicaResponseDataJsonConverter.write(((StopReplicaResponse) response).data(), version);
case SYNC_GROUP:
return SyncGroupResponseDataJsonConverter.write(((SyncGroupResponse) response).data(), version);
case TXN_OFFSET_COMMIT:
@@ -735,8 +705,6 @@ public class RequestConvertToJson {
return UnregisterBrokerResponseDataJsonConverter.write(((UnregisterBrokerResponse) response).data(), version);
case UPDATE_FEATURES:
return UpdateFeaturesResponseDataJsonConverter.write(((UpdateFeaturesResponse) response).data(), version);
- case UPDATE_METADATA:
- return UpdateMetadataResponseDataJsonConverter.write(((UpdateMetadataResponse) response).data(), version);
case VOTE:
return VoteResponseDataJsonConverter.write(((VoteResponse) response).data(), version);
case WRITE_SHARE_GROUP_STATE:
diff --git a/server/src/test/java/org/apache/kafka/network/RequestConvertToJsonTest.java b/server/src/test/java/org/apache/kafka/network/RequestConvertToJsonTest.java
index 56c5a822432..3bcb5136699 100644
--- a/server/src/test/java/org/apache/kafka/network/RequestConvertToJsonTest.java
+++ b/server/src/test/java/org/apache/kafka/network/RequestConvertToJsonTest.java
@@ -47,24 +47,27 @@ public class RequestConvertToJsonTest {
public void testAllRequestTypesHandled() {
List<String> unhandledKeys = new ArrayList<>();
for (ApiKeys key : ApiKeys.values()) {
- short version = key.latestVersion();
- ApiMessage message;
- if (key == ApiKeys.DESCRIBE_ACLS) {
- message = ApiMessageType.fromApiKey(key.id).newRequest();
- DescribeAclsRequestData requestData = (DescribeAclsRequestData) message;
- requestData.setPatternTypeFilter((byte) 1);
- requestData.setResourceTypeFilter((byte) 1);
- requestData.setPermissionType((byte) 1);
- requestData.setOperation((byte) 1);
- } else {
- message = ApiMessageType.fromApiKey(key.id).newRequest();
- }
- ByteBuffer bytes = MessageUtil.toByteBuffer(message, version);
- AbstractRequest req = AbstractRequest.parseRequest(key, version, bytes).request;
- try {
- RequestConvertToJson.request(req);
- } catch (IllegalStateException e) {
- unhandledKeys.add(key.toString());
+ if (key.hasValidVersion()) {
+ short version = key.latestVersion();
+ ApiMessage message;
+ if (key == ApiKeys.DESCRIBE_ACLS) {
+ message = ApiMessageType.fromApiKey(key.id).newRequest();
+ DescribeAclsRequestData requestData = (DescribeAclsRequestData) message;
+ requestData.setPatternTypeFilter((byte) 1);
+ requestData.setResourceTypeFilter((byte) 1);
+ requestData.setPermissionType((byte) 1);
+ requestData.setOperation((byte) 1);
+ } else {
+ message = ApiMessageType.fromApiKey(key.id).newRequest();
+ }
+ ByteBuffer bytes = MessageUtil.toByteBuffer(message, version);
+ AbstractRequest req = AbstractRequest.parseRequest(key, version, bytes).request;
+ try {
+ RequestConvertToJson.request(req);
+ } catch (IllegalStateException e) {
+ e.printStackTrace();
+ unhandledKeys.add(key.toString());
+ }
}
}
assertEquals(Collections.emptyList(), unhandledKeys, "Unhandled request keys");
@@ -101,14 +104,16 @@ public class RequestConvertToJsonTest {
public void testAllResponseTypesHandled() {
List<String> unhandledKeys = new ArrayList<>();
for (ApiKeys key : ApiKeys.values()) {
- short version = key.latestVersion();
- ApiMessage message = ApiMessageType.fromApiKey(key.id).newResponse();
- ByteBuffer bytes = MessageUtil.toByteBuffer(message, version);
- AbstractResponse res = AbstractResponse.parseResponse(key, bytes, version);
- try {
- RequestConvertToJson.response(res, version);
- } catch (IllegalStateException e) {
- unhandledKeys.add(key.toString());
+ if (key.hasValidVersion()) {
+ short version = key.latestVersion();
+ ApiMessage message = ApiMessageType.fromApiKey(key.id).newResponse();
+ ByteBuffer bytes = MessageUtil.toByteBuffer(message, version);
+ AbstractResponse res = AbstractResponse.parseResponse(key, bytes, version);
+ try {
+ RequestConvertToJson.response(res, version);
+ } catch (IllegalStateException e) {
+ unhandledKeys.add(key.toString());
+ }
}
}
assertEquals(Collections.emptyList(), unhandledKeys, "Unhandled response keys");
diff --git a/tools/src/test/java/org/apache/kafka/tools/TopicCommandTest.java b/tools/src/test/java/org/apache/kafka/tools/TopicCommandTest.java
index b35952ee088..bde7f3f2532 100644
--- a/tools/src/test/java/org/apache/kafka/tools/TopicCommandTest.java
+++ b/tools/src/test/java/org/apache/kafka/tools/TopicCommandTest.java
@@ -44,7 +44,6 @@ import org.apache.kafka.common.errors.ClusterAuthorizationException;
import org.apache.kafka.common.errors.ThrottlingQuotaExceededException;
import org.apache.kafka.common.errors.TopicExistsException;
import org.apache.kafka.common.internals.Topic;
-import org.apache.kafka.common.message.UpdateMetadataRequestData;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.MetadataResponse;
@@ -57,6 +56,7 @@ import org.apache.kafka.common.test.api.ClusterTest;
import org.apache.kafka.common.test.api.ClusterTestExtensions;
import org.apache.kafka.common.test.api.Type;
import org.apache.kafka.common.utils.Exit;
+import org.apache.kafka.metadata.LeaderAndIsr;
import org.apache.kafka.server.common.AdminCommandFailedException;
import org.apache.kafka.server.common.AdminOperationException;
import org.apache.kafka.test.TestUtils;
@@ -1112,8 +1112,8 @@ public class TopicCommandTest {
TestUtils.waitForCondition(
() -> clusterInstance.aliveBrokers().values().stream().allMatch(
broker -> {
- Optional<UpdateMetadataRequestData.UpdateMetadataPartitionState> partitionState =
- Optional.ofNullable(broker.metadataCache().getPartitionInfo(testTopicName, 0).getOrElse(null));
+ Optional<LeaderAndIsr> partitionState = Optional.ofNullable(
+ broker.metadataCache().getLeaderAndIsr(testTopicName, 0).getOrElse(null));
return partitionState.map(s -> FetchRequest.isValidBrokerId(s.leader())).orElse(false);
}
), CLUSTER_WAIT_MS, String.format("Meta data propogation fail in %s ms", CLUSTER_WAIT_MS));
@@ -1141,7 +1141,7 @@ public class TopicCommandTest {
assertEquals(2, clusterInstance.aliveBrokers().size());
TestUtils.waitForCondition(
- () -> clusterInstance.aliveBrokers().values().stream().allMatch(broker -> broker.metadataCache().getPartitionInfo(testTopicName, 0).get().isr().size() == 2),
+ () -> clusterInstance.aliveBrokers().values().stream().allMatch(broker -> broker.metadataCache().getLeaderAndIsr(testTopicName, 0).get().isr().size() == 2),
CLUSTER_WAIT_MS, String.format("Timeout waiting for partition metadata propagating to brokers for %s topic", testTopicName)
);
@@ -1244,7 +1244,7 @@ public class TopicCommandTest {
assertEquals(4, clusterInstance.aliveBrokers().size());
TestUtils.waitForCondition(
- () -> clusterInstance.aliveBrokers().values().stream().allMatch(broker -> broker.metadataCache().getPartitionInfo(testTopicName, 0).get().isr().size() == 4),
+ () -> clusterInstance.aliveBrokers().values().stream().allMatch(broker -> broker.metadataCache().getLeaderAndIsr(testTopicName, 0).get().isr().size() == 4),
CLUSTER_WAIT_MS, String.format("Timeout waiting for partition metadata propagating to brokers for %s topic", testTopicName)
);
@@ -1302,8 +1302,8 @@ public class TopicCommandTest {
TestUtils.waitForCondition(
() -> clusterInstance.aliveBrokers().values().stream().allMatch(broker ->
- broker.metadataCache().getPartitionInfo(underMinIsrTopic, 0).get().isr().size() < 6 &&
- broker.metadataCache().getPartitionInfo(offlineTopic, 0).get().leader() == MetadataResponse.NO_LEADER_ID),
+ broker.metadataCache().getLeaderAndIsr(underMinIsrTopic, 0).get().isr().size() < 6 &&
+ broker.metadataCache().getLeaderAndIsr(offlineTopic, 0).get().leader() == MetadataResponse.NO_LEADER_ID),
CLUSTER_WAIT_MS, "Timeout waiting for partition metadata propagating to brokers for underMinIsrTopic topic"
);