You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2021/03/05 07:37:28 UTC
[GitHub] [kafka] dajac commented on a change in pull request #10269: KAFKA-12410 KafkaAPis ought to group fetch data before generating fet…
dajac commented on a change in pull request #10269:
URL: https://github.com/apache/kafka/pull/10269#discussion_r588075100
##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -811,31 +821,40 @@ class KafkaApis(val requestChannel: RequestChannel,
.setRecords(data.records)
.setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))
data.divergingEpoch.foreach(partitionData.setDivergingEpoch)
- partitions.put(tp, partitionData)
+ addPartition(tp.topic, partitionData)
}
- erroneous.foreach { case (tp, data) => partitions.put(tp, data) }
-
- var unconvertedFetchResponse: FetchResponse = null
+ erroneous.foreach { case (tp, data) => addPartition(tp.topic, data) }
- def createResponse(throttleTimeMs: Int): FetchResponse = {
+ def createResponse(unconvertedFetchResponse: FetchResponse, throttleTimeMs: Int): FetchResponse = {
// Down-convert messages for each partition if required
- val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData]
- unconvertedFetchResponse.responseData.forEach { (tp, unconvertedPartitionData) =>
- val error = Errors.forCode(unconvertedPartitionData.errorCode)
- if (error != Errors.NONE)
- debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " +
- s"on partition $tp failed due to ${error.exceptionName}")
- convertedData.put(tp, maybeConvertFetchedData(tp, unconvertedPartitionData))
+ val convertedResponse = new FetchResponseData()
+ .setErrorCode(unconvertedFetchResponse.error.code)
+ .setThrottleTimeMs(throttleTimeMs)
+ .setSessionId(unconvertedFetchResponse.sessionId)
+ .setResponses(new util.ArrayList[FetchResponseData.FetchableTopicResponse](unconvertedFetchResponse.data.responses.size))
+ unconvertedFetchResponse.data.responses.forEach { unconvertedTopicData =>
+ val convertedTopicResponse = new FetchResponseData.FetchableTopicResponse()
+ .setTopic(unconvertedTopicData.topic)
+ .setPartitions(new util.ArrayList[FetchResponseData.PartitionData](unconvertedTopicData.partitions.size))
+ convertedResponse.responses.add(convertedTopicResponse)
+ unconvertedTopicData.partitions.forEach { unconvertedPartitionData =>
+ val tp = new TopicPartition(unconvertedTopicData.topic, unconvertedPartitionData.partitionIndex)
+ val error = Errors.forCode(unconvertedPartitionData.errorCode)
+ if (error != Errors.NONE)
Review comment:
Would it make sense to replace this by 'errorCode != Errors.NONE.code'?
##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -811,31 +821,40 @@ class KafkaApis(val requestChannel: RequestChannel,
.setRecords(data.records)
.setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))
data.divergingEpoch.foreach(partitionData.setDivergingEpoch)
- partitions.put(tp, partitionData)
+ addPartition(tp.topic, partitionData)
}
- erroneous.foreach { case (tp, data) => partitions.put(tp, data) }
-
- var unconvertedFetchResponse: FetchResponse = null
+ erroneous.foreach { case (tp, data) => addPartition(tp.topic, data) }
- def createResponse(throttleTimeMs: Int): FetchResponse = {
+ def createResponse(unconvertedFetchResponse: FetchResponse, throttleTimeMs: Int): FetchResponse = {
// Down-convert messages for each partition if required
- val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData]
- unconvertedFetchResponse.responseData.forEach { (tp, unconvertedPartitionData) =>
- val error = Errors.forCode(unconvertedPartitionData.errorCode)
- if (error != Errors.NONE)
- debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " +
- s"on partition $tp failed due to ${error.exceptionName}")
- convertedData.put(tp, maybeConvertFetchedData(tp, unconvertedPartitionData))
+ val convertedResponse = new FetchResponseData()
Review comment:
Could we mutate the response instead of creating a new one?
##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -811,31 +821,40 @@ class KafkaApis(val requestChannel: RequestChannel,
.setRecords(data.records)
.setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))
data.divergingEpoch.foreach(partitionData.setDivergingEpoch)
- partitions.put(tp, partitionData)
+ addPartition(tp.topic, partitionData)
}
- erroneous.foreach { case (tp, data) => partitions.put(tp, data) }
-
- var unconvertedFetchResponse: FetchResponse = null
+ erroneous.foreach { case (tp, data) => addPartition(tp.topic, data) }
- def createResponse(throttleTimeMs: Int): FetchResponse = {
+ def createResponse(unconvertedFetchResponse: FetchResponse, throttleTimeMs: Int): FetchResponse = {
// Down-convert messages for each partition if required
- val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData]
- unconvertedFetchResponse.responseData.forEach { (tp, unconvertedPartitionData) =>
- val error = Errors.forCode(unconvertedPartitionData.errorCode)
- if (error != Errors.NONE)
- debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " +
- s"on partition $tp failed due to ${error.exceptionName}")
- convertedData.put(tp, maybeConvertFetchedData(tp, unconvertedPartitionData))
+ val convertedResponse = new FetchResponseData()
+ .setErrorCode(unconvertedFetchResponse.error.code)
+ .setThrottleTimeMs(throttleTimeMs)
+ .setSessionId(unconvertedFetchResponse.sessionId)
+ .setResponses(new util.ArrayList[FetchResponseData.FetchableTopicResponse](unconvertedFetchResponse.data.responses.size))
+ unconvertedFetchResponse.data.responses.forEach { unconvertedTopicData =>
+ val convertedTopicResponse = new FetchResponseData.FetchableTopicResponse()
+ .setTopic(unconvertedTopicData.topic)
+ .setPartitions(new util.ArrayList[FetchResponseData.PartitionData](unconvertedTopicData.partitions.size))
+ convertedResponse.responses.add(convertedTopicResponse)
+ unconvertedTopicData.partitions.forEach { unconvertedPartitionData =>
+ val tp = new TopicPartition(unconvertedTopicData.topic, unconvertedPartitionData.partitionIndex)
Review comment:
It's a pity that we have te recreate TopicPartition object here. We have to do the same in couple of other places... I am not sure about the impact though. What's your take on this?
I wonder if we could cache it in the response. We could extend the automated protocol to support "transient fields" (a field which is declared but not serialized) and store it there.
##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -811,31 +821,40 @@ class KafkaApis(val requestChannel: RequestChannel,
.setRecords(data.records)
.setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))
data.divergingEpoch.foreach(partitionData.setDivergingEpoch)
- partitions.put(tp, partitionData)
+ addPartition(tp.topic, partitionData)
}
- erroneous.foreach { case (tp, data) => partitions.put(tp, data) }
-
- var unconvertedFetchResponse: FetchResponse = null
+ erroneous.foreach { case (tp, data) => addPartition(tp.topic, data) }
Review comment:
forKeyValue?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org