You are viewing a plain text version of this content. The canonical link for it is here.

Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2020/10/29 03:34:10 UTC

[GitHub] [kafka] anatasiavela opened a new pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

anatasiavela opened a new pull request #9526:
URL: https://github.com/apache/kafka/pull/9526


   Kafka’s request and response traces currently output in a format that is JSON-like and are not easily parsable.
   
   There is a new auto-generated schema for each request type that supports outputting JSON payloads for request and response payloads. These can be adapted to provide structured request tracing.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514574814



##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -162,7 +161,7 @@ object RequestChannel extends Logging {
       }
     }
 
-    trace(s"Processor $processor received request: ${requestDesc(true)}")
+    trace(s"Processor $processor received request: ${RequestConvertToJson.requestDesc(header, loggableRequest, true).toString}")

Review comment:
       Would we unnecessarily calculate it here? I think the macro will avoid it in this case, but I'm fine with gating it anyway.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514576105



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))

Review comment:
       Yes, I agree, this would be helpful if it's not too difficult.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r526265466



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version, false)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = requestDesc(header, req).asInstanceOf[ObjectNode]
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, serializeRecords: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null) {
+          partitionNode.set("records", NullNode.instance)
+        } else {
+          if (serializeRecords)
+            partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))
+          else
+            partitionNode.set("records", new IntNode(partitionData.validBytes()))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochResponseNode(response: OffsetsForLeaderEpochResponse, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 2) {
+      node.set("throttleTimeMs", new IntNode(response.throttleTimeMs))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(response.responses)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("errorCode", new ShortNode(partitionData.error.code))
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 1) partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionNode.set("endOffset", new LongNode(partitionData.endOffset))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceResponseNode(response: ProduceResponse, version: Short): JsonNode = {

Review comment:
       Great




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517699783



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       Nvm, we can't deserialize `recordSet` to an IntNode because `FetchResponseData` expects `recordSet` to be of type `BaseRecords`, not `int`.
   What I've done was left deserialization the same and added the verbose tag to serialization.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517651742



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       Just read through the comment below, so I'll keep the `verbose` flag for now, but I'll remove it depending on what we decide below.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517651742



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       Just read through the comment below, so I'll keep it for now.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518359188



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version, verbose)
+      case req: ProduceRequest => produceRequestNode(req, request.version, verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short, verbose: Boolean): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version, verbose)
+      case res: ProduceResponse => produceResponseNode(res, version, verbose)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader, verbose: Boolean): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), verbose).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,

Review comment:
       I'm not too sure what this should test, since we don't have an expected output for this besides all fields being filled. What do you think we should test?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r526258381



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version, false)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = requestDesc(header, req).asInstanceOf[ObjectNode]
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, serializeRecords: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null) {
+          partitionNode.set("records", NullNode.instance)
+        } else {
+          if (serializeRecords)
+            partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))
+          else
+            partitionNode.set("records", new IntNode(partitionData.validBytes()))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochResponseNode(response: OffsetsForLeaderEpochResponse, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 2) {
+      node.set("throttleTimeMs", new IntNode(response.throttleTimeMs))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(response.responses)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("errorCode", new ShortNode(partitionData.error.code))
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 1) partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionNode.set("endOffset", new LongNode(partitionData.endOffset))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceResponseNode(response: ProduceResponse, version: Short): JsonNode = {

Review comment:
       We are actually waiting on that one and this one https://github.com/apache/kafka/pull/9547.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514619891



##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       I guess this is related with the inconsistency issue, but I tried not to make too many changes. The constructors with the `data` parameters were private in some cases, so I used the `toStruct`. But for consistency, I'll change to use the Builder class instead.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517665383



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @lbradstreet The current generated JSON does not print the recordSet. When we were serializing a BinaryNode with an empty array, it was still being deserialized as a BinaryNode, so it doesn't break anything.
   ```
   buffer.printf("%s;%n", target.assignmentStatement(
                   String.format("MemoryRecords.readableRecords(ByteBuffer.wrap(MessageUtil.jsonNodeToBinary(%s, \"%s\")))",
                       target.sourceVariable(), target.humanReadableName())));
   ```
   So I believe the concern is that I had proposed to change the serialization to an IntNode, but the deserialization still expects a BinaryNode which I should have also changed.
   Another alternative I could fix this by changing the deserialization to expect an IntNode, but I'm for adding a verbose flag
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r516288558



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       I changed the `recordSet` autogenerated JSON output to be the number of bytes, instead of an empty byte array. 




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517649219



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null)
+          partitionNode.set("records", NullNode.instance)
+        else
+          partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))

Review comment:
       The `ProduceRequest` does not match the conditions for the change made in the automated protocol. The `records` field is of type `bytes` in `ProduceRequest`, whereas the change to the protocol only effects fields of type `records` like in `FetchResponse`.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514093626



##########
File path: clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java
##########
@@ -57,6 +57,10 @@ public AddOffsetsToTxnRequest(Struct struct, short version) {
         this.data = new AddOffsetsToTxnRequestData(struct, version);
     }
 
+    public AddOffsetsToTxnRequestData data() {
+        return data;
+    }

Review comment:
       This is not entirely related to your PR but it seems that we are little inconsistent wrt. how we expose/access `data` in the code base. As an example, here we add an accessor to access `data` but we keep `data` field public and keep accessing it directly from various places.
   
   I think that we should do one the following:
   1) Add an accessor and make `data` final private everywhere; or
   2) Make `data` final public everywhere.
   
   It seems that you suggest to go with 1) in the PR. Personally, I don't have a strong preference for any of them but I would like us to be consistent across the board.

##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -162,7 +161,7 @@ object RequestChannel extends Logging {
       }
     }
 
-    trace(s"Processor $processor received request: ${requestDesc(true)}")
+    trace(s"Processor $processor received request: ${RequestConvertToJson.requestDesc(header, loggableRequest, true).toString}")

Review comment:
       We may want to gate this by `isTraceEnabled` in order to not compute it necessarily.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)

Review comment:
       This is one case where we still access `data` without an accessor.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())

Review comment:
       It is a little frustrating to have to maintain this mapping for each request. I wonder if we could auto-generate such helper based on the protocol description. We already generate `DataJsonConverter` helpers so it should be feasible to also generate an overall helper.
   
   Have you already considered this? I am fine with keep it as-is for now though and to investigate this in the future.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))

Review comment:
       I wonder if we should also structure the client information in JSON instead of relying on the string representation of the object. That would make the information more accessible.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {

Review comment:
       Should we add unit tests for these helper methods? I mean that it would be great to ensure that the format of the JSON is equivalent to the JSON what will be generated by the auto-generated protocol.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    if (verbose) {
+      val partSizes = new ArrayNode(JsonNodeFactory.instance)
+      for (partSize <- request.partitionSizes().entrySet.asScala) {
+        val part = new ObjectNode(JsonNodeFactory.instance)
+        val topic = partSize.getKey
+        part.set("partition", new TextNode(topic.toString))
+        part.set("size", new IntNode(partSize.getValue))
+        partSizes.add(part)
+      }
+      node.set("partitionSizes", partSizes)
+    } else {
+      node.set("partitionSizes", new IntNode(request.partitionSizes.size))

Review comment:
       It is a bit weird to have `partitionSizes` used with two different type of value. It seems that we were previously using `numPartitions` for this case.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       It is important to note that the format of the JSON will change when we move to the generated schema. `partitionSizes` does not exist in the schema. Thus, I wonder if we should better align directly; or keep this exception forever. Have we thought about this?
   
   Similarly in the `FetchResponse`, we were outputting `recordsSizeInBytes` per partition and it seems that now we put an empty `recordSet` field. cc @lbradstreet 

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {

Review comment:
       nit: Could we align them with `header`?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }

Review comment:
       Should we handle the unknown cases explicitly and throw an exception with an meaningful message instead of relying on `MatchError`? What do you think?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))

Review comment:
       nit: Could we add a comment here to explain why we overwrite the `requestApiKey`?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       I am working on this one :)

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    if (verbose) {
+      val partSizes = new ArrayNode(JsonNodeFactory.instance)
+      for (partSize <- request.partitionSizes().entrySet.asScala) {

Review comment:
       nit: We may be able to use `request.partitionSizes().forEach` directly.

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       Why are we using `toStruct` in some cases and passing the `*Data` object directly in others?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    if (verbose) {
+      val partSizes = new ArrayNode(JsonNodeFactory.instance)
+      for (partSize <- request.partitionSizes().entrySet.asScala) {
+        val part = new ObjectNode(JsonNodeFactory.instance)
+        val topic = partSize.getKey
+        part.set("partition", new TextNode(topic.toString))

Review comment:
       We are using the `topic` for the `partition` here. I suppose that we need both, isn't it?

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)
+    case ApiKeys.SYNC_GROUP => new SyncGroupRequest(new SyncGroupRequestData(), version)
+    case ApiKeys.STOP_REPLICA => new StopReplicaRequest(new StopReplicaRequestData().toStruct(version), version)
+    case ApiKeys.CONTROLLED_SHUTDOWN => new ControlledShutdownRequest(new ControlledShutdownRequestData().toStruct(version), version)
+    case ApiKeys.UPDATE_METADATA => new UpdateMetadataRequest(new UpdateMetadataRequestData().toStruct(version), version)
+    case ApiKeys.LEADER_AND_ISR => new LeaderAndIsrRequest(new LeaderAndIsrRequestData().toStruct(version), version)
+    case ApiKeys.DESCRIBE_GROUPS => new DescribeGroupsRequest(new DescribeGroupsRequestData().toStruct(version), version)
+    case ApiKeys.LIST_GROUPS => new ListGroupsRequest(new ListGroupsRequestData(), version)
+    case ApiKeys.SASL_HANDSHAKE => new SaslHandshakeRequest(new SaslHandshakeRequestData())
+    case ApiKeys.API_VERSIONS => new ApiVersionsRequest(new ApiVersionsRequestData(), version)
+    case ApiKeys.CREATE_TOPICS => new CreateTopicsRequest(new CreateTopicsRequestData().toStruct(version), version)
+    case ApiKeys.DELETE_TOPICS => new DeleteTopicsRequest(new DeleteTopicsRequestData().toStruct(version), version)
+    case ApiKeys.DELETE_RECORDS => new DeleteRecordsRequest(new DeleteRecordsRequestData().toStruct(version), version)
+    case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdRequest(new InitProducerIdRequestData().toStruct(version), version)
+    case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochRequest(new OffsetForLeaderEpochResponseData().toStruct(version), version)
+    case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnRequest(new AddPartitionsToTxnRequestData(), version)
+    case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnRequest(new AddOffsetsToTxnRequestData(), version)
+    case ApiKeys.END_TXN =>  new EndTxnRequest(new EndTxnRequestData().toStruct(version), version)
+    case ApiKeys.WRITE_TXN_MARKERS =>  new WriteTxnMarkersRequest(new WriteTxnMarkersRequestData().toStruct(version), version)
+    case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitRequest(new TxnOffsetCommitRequestData(), version)
+    case ApiKeys.DESCRIBE_ACLS =>
+      val data = new DescribeAclsRequestData().setResourceTypeFilter(1).setOperation(2).setPermissionType(2)
+      new DescribeAclsRequest(data.toStruct(version), version)
+    case ApiKeys.CREATE_ACLS =>  new CreateAclsRequest(new CreateAclsRequestData().toStruct(version), version)
+    case ApiKeys.DELETE_ACLS => new DeleteAclsRequest(new DeleteAclsRequestData().toStruct(version), version)
+    case ApiKeys.DESCRIBE_CONFIGS => new DescribeConfigsRequest(new DescribeConfigsRequestData(), version)
+    case ApiKeys.ALTER_CONFIGS => new AlterConfigsRequest(new AlterConfigsRequestData(), version)
+    case ApiKeys.ALTER_REPLICA_LOG_DIRS =>  new AlterReplicaLogDirsRequest(new AlterReplicaLogDirsRequestData(), version)
+    case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsRequest(new DescribeLogDirsRequestData(), version)
+    case ApiKeys.SASL_AUTHENTICATE => new SaslAuthenticateRequest(new SaslAuthenticateRequestData(), version)
+    case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsRequest(new CreatePartitionsRequestData().toStruct(version), version)
+    case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenRequest(new CreateDelegationTokenRequestData().toStruct(version), version)
+    case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenRequest(new RenewDelegationTokenRequestData(), version)
+    case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenRequest(new ExpireDelegationTokenRequestData().toStruct(version), version)
+    case ApiKeys.DESCRIBE_DELEGATION_TOKEN => new DescribeDelegationTokenRequest(new DescribeDelegationTokenRequestData(), version)
+    case ApiKeys.DELETE_GROUPS => new DeleteGroupsRequest(new DeleteGroupsRequestData(), version)
+    case ApiKeys.ELECT_LEADERS => new ElectLeadersRequest(new ElectLeadersRequestData().toStruct(version), version)
+    case ApiKeys.INCREMENTAL_ALTER_CONFIGS => new IncrementalAlterConfigsRequest.Builder(new IncrementalAlterConfigsRequestData()).build(version)
+    case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsRequest.Builder(new AlterPartitionReassignmentsRequestData()).build(version)
+    case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsRequest.Builder(new ListPartitionReassignmentsRequestData()).build(version)
+    case ApiKeys.OFFSET_DELETE => new OffsetDeleteRequest(new OffsetDeleteRequestData(), version)
+    case ApiKeys.DESCRIBE_CLIENT_QUOTAS => new DescribeClientQuotasRequest(new DescribeClientQuotasRequestData(), version)
+    case ApiKeys.ALTER_CLIENT_QUOTAS => new AlterClientQuotasRequest(new AlterClientQuotasRequestData(), version)
+    case ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS => new DescribeUserScramCredentialsRequest.Builder(new DescribeUserScramCredentialsRequestData()).build(version)
+    case ApiKeys.ALTER_USER_SCRAM_CREDENTIALS => new AlterUserScramCredentialsRequest.Builder(new AlterUserScramCredentialsRequestData()).build(version)
+    case ApiKeys.VOTE => new VoteRequest.Builder(new VoteRequestData()).build(version)
+    case ApiKeys.BEGIN_QUORUM_EPOCH => new BeginQuorumEpochRequest.Builder(new BeginQuorumEpochRequestData()).build(version)
+    case ApiKeys.END_QUORUM_EPOCH => new EndQuorumEpochRequest.Builder(new EndQuorumEpochRequestData()).build(version)
+    case ApiKeys.DESCRIBE_QUORUM => new DescribeQuorumRequest.Builder(new DescribeQuorumRequestData()).build(version)
+    case ApiKeys.ALTER_ISR => new AlterIsrRequest.Builder(new AlterIsrRequestData()).build(version)
+    case ApiKeys.UPDATE_FEATURES => new UpdateFeaturesRequest.Builder(new UpdateFeaturesRequestData()).build(version)
+    case _ => throw new AssertionError(String.format("Request type %s is not tested in `RequestConvertToJsonTest`", apiKey))
+  }
+
+  def createResponseFromApiKey(apiKey: ApiKeys, version: Short): AbstractResponse = apiKey match {
+    case ApiKeys.PRODUCE => new ProduceResponse(new ProduceResponseData().toStruct(version))
+    case ApiKeys.FETCH => new FetchResponse(new FetchResponseData())
+    case ApiKeys.LIST_OFFSETS => new ListOffsetResponse(new ListOffsetResponseData())
+    case ApiKeys.METADATA => new MetadataResponse(new MetadataResponseData())
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitResponse(new OffsetCommitResponseData())
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(new OffsetFetchResponseData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorResponse(new FindCoordinatorResponseData())
+    case ApiKeys.JOIN_GROUP => new JoinGroupResponse(new JoinGroupResponseData())
+    case ApiKeys.HEARTBEAT => new HeartbeatResponse(new HeartbeatResponseData())
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupResponse(new LeaveGroupResponseData())
+    case ApiKeys.SYNC_GROUP => new SyncGroupResponse(new SyncGroupResponseData())
+    case ApiKeys.STOP_REPLICA => new StopReplicaResponse(new StopReplicaResponseData())
+    case ApiKeys.CONTROLLED_SHUTDOWN => new ControlledShutdownResponse(new ControlledShutdownResponseData())
+    case ApiKeys.UPDATE_METADATA => new UpdateMetadataResponse(new UpdateMetadataResponseData())
+    case ApiKeys.LEADER_AND_ISR => new LeaderAndIsrResponse(new LeaderAndIsrResponseData())
+    case ApiKeys.DESCRIBE_GROUPS => new DescribeGroupsResponse(new DescribeGroupsResponseData())
+    case ApiKeys.LIST_GROUPS => new ListGroupsResponse(new ListGroupsResponseData())
+    case ApiKeys.SASL_HANDSHAKE => new SaslHandshakeResponse(new SaslHandshakeResponseData())
+    case ApiKeys.API_VERSIONS => new ApiVersionsResponse(new ApiVersionsResponseData())
+    case ApiKeys.CREATE_TOPICS => new CreateTopicsResponse(new CreateTopicsResponseData())
+    case ApiKeys.DELETE_TOPICS => new DeleteTopicsResponse(new DeleteTopicsResponseData())
+    case ApiKeys.DELETE_RECORDS => new DeleteRecordsResponse(new DeleteRecordsResponseData())
+    case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdResponse(new InitProducerIdResponseData())
+    case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(new OffsetForLeaderEpochResponseData().toStruct(version))
+    case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnResponse(new AddPartitionsToTxnResponseData().toStruct(version), version)
+    case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnResponse(new AddOffsetsToTxnResponseData())
+    case ApiKeys.END_TXN => new EndTxnResponse(new EndTxnResponseData())
+    case ApiKeys.WRITE_TXN_MARKERS => new WriteTxnMarkersResponse(new WriteTxnMarkersResponseData().toStruct(version), version)
+    case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(new TxnOffsetCommitResponseData())
+    case ApiKeys.DESCRIBE_ACLS => new DescribeAclsResponse(new DescribeAclsResponseData())
+    case ApiKeys.CREATE_ACLS => new CreateAclsResponse(new CreateAclsResponseData())
+    case ApiKeys.DELETE_ACLS => new DeleteAclsResponse(new DeleteAclsResponseData())
+    case ApiKeys.DESCRIBE_CONFIGS => new DescribeConfigsResponse(new DescribeConfigsResponseData())
+    case ApiKeys.ALTER_CONFIGS => new AlterConfigsResponse(new AlterConfigsResponseData())
+    case ApiKeys.ALTER_REPLICA_LOG_DIRS => new AlterReplicaLogDirsResponse(new AlterReplicaLogDirsResponseData())
+    case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(new DescribeLogDirsResponseData())
+    case ApiKeys.SASL_AUTHENTICATE => new SaslAuthenticateResponse(new SaslAuthenticateResponseData())
+    case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(new CreatePartitionsResponseData())
+    case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(new CreateDelegationTokenResponseData())
+    case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(new RenewDelegationTokenResponseData())
+    case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(new ExpireDelegationTokenResponseData())
+    case ApiKeys.DESCRIBE_DELEGATION_TOKEN => new DescribeDelegationTokenResponse(new DescribeDelegationTokenResponseData().toStruct(version), version)
+    case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(new DeleteGroupsResponseData())
+    case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(new ElectLeadersResponseData().toStruct(version), version)
+    case ApiKeys.INCREMENTAL_ALTER_CONFIGS => new IncrementalAlterConfigsResponse(new IncrementalAlterConfigsResponseData())
+    case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(new AlterPartitionReassignmentsResponseData())
+    case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(new ListPartitionReassignmentsResponseData())
+    case ApiKeys.OFFSET_DELETE => new OffsetDeleteResponse(new OffsetDeleteResponseData())
+    case ApiKeys.DESCRIBE_CLIENT_QUOTAS => new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData().toStruct(version), version)
+    case ApiKeys.ALTER_CLIENT_QUOTAS => new AlterClientQuotasResponse(new AlterClientQuotasResponseData().toStruct(version), version)
+    case ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS => new DescribeUserScramCredentialsResponse(new DescribeUserScramCredentialsResponseData())
+    case ApiKeys.ALTER_USER_SCRAM_CREDENTIALS => new AlterUserScramCredentialsResponse(new AlterUserScramCredentialsResponseData())
+    case ApiKeys.VOTE => new VoteResponse(new VoteResponseData())
+    case ApiKeys.BEGIN_QUORUM_EPOCH => new BeginQuorumEpochResponse(new BeginQuorumEpochResponseData())
+    case ApiKeys.END_QUORUM_EPOCH => new EndQuorumEpochResponse(new EndQuorumEpochResponseData())
+    case ApiKeys.DESCRIBE_QUORUM => new DescribeQuorumResponse(new DescribeQuorumResponseData())
+    case ApiKeys.ALTER_ISR => new AlterIsrResponse(new AlterIsrResponseData())
+    case ApiKeys.UPDATE_FEATURES => new UpdateFeaturesResponse(new UpdateFeaturesResponseData())
+    case _ => throw new AssertionError(String.format("Response type %s not tested in `RequestConvertToJsonTest`", apiKey))
+  }
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = 0
+      val req = createRequestsFromApiKey(key, version)
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : Throwable => unhandledKeys += key.toString

Review comment:
       Shouldn't we only catch `AssertionError` here and below?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514861389



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    if (verbose) {
+      val partSizes = new ArrayNode(JsonNodeFactory.instance)
+      for (partSize <- request.partitionSizes().entrySet.asScala) {
+        val part = new ObjectNode(JsonNodeFactory.instance)
+        val topic = partSize.getKey
+        part.set("partition", new TextNode(topic.toString))

Review comment:
       Changed to match the auto-generated schema.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514508755



##########
File path: clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java
##########
@@ -57,6 +57,10 @@ public AddOffsetsToTxnRequest(Struct struct, short version) {
         this.data = new AddOffsetsToTxnRequestData(struct, version);
     }
 
+    public AddOffsetsToTxnRequestData data() {
+        return data;
+    }

Review comment:
       Yea I definitely agree with wanting consistency. I'll stick to 1) in the PR.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514577992



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))

Review comment:
       I was thinking a bit more about the time fields, and I think we should add `Ms` to the end of all of them. What do you think?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514617626



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       Yay! Excited for this to make even more consistent with other request types. 




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514575372



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))

Review comment:
       Having thought about it a bit more, maybe we could include both the api key number and the api key name rather than overwriting it?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514872477



##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       Actually, I found a workaround for this mapping and removed it all together, so there'd be one less apikey mapping to maintain.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514577021



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       I think that is fair, it would be good to align now. With the FetchResponse I would like to circle back on it and provide more detail. If not the recordSizeInBytes in the FetchResponse I think we need to include a top level responseSizeBytes in the trace.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517665383



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @lbradstreet The current generated JSON does not print the recordSet either. But when we are serializing a BinaryNode with an empty array, it is still being deserialized as a BinaryNode, so it doesn't break anything.
   ```
   buffer.printf("%s;%n", target.assignmentStatement(
                   String.format("MemoryRecords.readableRecords(ByteBuffer.wrap(MessageUtil.jsonNodeToBinary(%s, \"%s\")))",
                       target.sourceVariable(), target.humanReadableName())));
   ```
   So I believe the concern is that I had proposed to change the serialization to an IntNode, but the deserialization still expects a BinaryNode which I should have also changed.
   Another alternative I could fix this by changing the deserialization to expect an IntNode, but I'm for adding a verbose flag
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r542308308



##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -123,9 +131,9 @@ object RequestChannel extends Logging {
       }
     }
 
-    def responseString(response: AbstractResponse): Option[String] = {
+    def responseNode(response: AbstractResponse): Option[JsonNode] = {
       if (RequestChannel.isRequestLoggingEnabled)
-        Some(response.toString)
+        Some(RequestConvertToJson.response(response, context.apiVersion()))

Review comment:
       nit: `context.apiVersion()` -> `context.apiVersion`

##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -97,7 +98,8 @@ import scala.annotation.nowarn
 /**
  * Logic to handle the various Kafka requests
  */
-class KafkaApis(val requestChannel: RequestChannel,
+class
+KafkaApis(val requestChannel: RequestChannel,

Review comment:
       This is likely a mistake.

##########
File path: core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala
##########
@@ -39,7 +40,8 @@ class TestRaftRequestHandler(
 
   override def handle(request: RequestChannel.Request): Unit = {
     try {
-      trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" +
+      trace(s"Handling request:${RequestConvertToJson.requestDesc(request.header, request.requestLog, request.isForwarded)} " +

Review comment:
       We should also revert this one.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, JsonNodeFactory, LongNode, ObjectNode, TextNode}
+import kafka.network.RequestChannel.Session
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.requests._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => OffsetForLeaderEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ProduceRequest => ProduceRequestDataJsonConverter.write(req.data, request.version, false)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.apiKey} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => OffsetForLeaderEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ProduceResponse => ProduceResponseDataJsonConverter.write(res.data, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey ${response.apiKey} is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data, header.headerVersion, false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestContextNode(context: RequestContext): JsonNode = {

Review comment:
       It seems that this one is not used anymore.

##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -183,7 +185,8 @@ class KafkaApis(val requestChannel: RequestChannel,
    */
   override def handle(request: RequestChannel.Request): Unit = {
     try {
-      trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" +
+      trace(s"Handling request:${RequestConvertToJson.requestDesc(request.header, request.requestLog, request.isForwarded)} " +

Review comment:
       We should also revert this one.

##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -200,7 +208,7 @@ object RequestChannel extends Logging {
       }
     }
 
-    trace(s"Processor $processor received request: ${requestDesc(true)}")
+    trace(s"Processor $processor received request: ${RequestConvertToJson.requestDesc(header, requestLog, isForwarded)}")

Review comment:
       As discussed offline, I think that we should revert this one. We want to use JSON in the request log (aka `requestLogger`), not as a replacement of all the `toString` usages.

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+
+import com.fasterxml.jackson.databind.node.{DoubleNode, LongNode, ObjectNode, TextNode}
+import kafka.network
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, ByteBufferAccessor, ObjectSerializationCache}
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach { key => {
+      val version: Short = key.latestVersion()
+      val cache = new ObjectSerializationCache
+      val message = key match {
+        case ApiKeys.DESCRIBE_ACLS =>
+          ApiMessageType.fromApiKey(key.id).newRequest().asInstanceOf[DescribeAclsRequestData]
+            .setPatternTypeFilter(1).setResourceTypeFilter(1).setPermissionType(1).setOperation(1)
+        case _ =>
+          ApiMessageType.fromApiKey(key.id).newRequest()
+      }
+      val messageSize = message.size(cache, version)
+      val bytes = new ByteBufferAccessor(ByteBuffer.allocate(messageSize))
+      message.write(bytes, cache, version)
+      bytes.flip()
+
+      val req = AbstractRequest.parseRequest(key, version, bytes.buffer).request
+      try {
+        RequestConvertToJson.request(req)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    }}
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+

Review comment:
       Could we also add unit tests for `RequestConvertToJson.requestDesc`, `RequestConvertToJson.requestHeaderNode` and `RequestConvertToJson.clientInfoNode`?

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -377,11 +387,24 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
                 buffer.printf("%s;%n", target.assignmentStatement(
                     String.format("new BinaryNode(Arrays.copyOf(%s, %s.length))",
-                        target.sourceVariable(), target.sourceVariable())));
+                            target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
             headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            // KIP-673: When logging requests/responses, we do not serialize the record, instead we
+            // output its sizeInBytes, because outputting the bytes is not very useful and can be
+            // quite expensive. Otherwise, we will serialize the record.
+            buffer.printf("if (_serializeRecords) {%n");
+            buffer.incrementIndent();
             buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            buffer.decrementIndent();
+            buffer.printf("} else {%n");
+            buffer.incrementIndent();
+            buffer.printf("%s;%n", target.assignmentStatement(

Review comment:
       Would something like the following work?
   ```
   buffer.printf("_node.set(\"%sSizeInBytes\", new IntNode(%s.sizeInBytes()));%n",
     target.field().camelCaseName(),
     target.sourceVariable());
   ```




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac merged pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac merged pull request #9526:
URL: https://github.com/apache/kafka/pull/9526


   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514635784



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))

Review comment:
       How about requestApi or requestApiName?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517317354



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       It seems that `verbose` is not used anymore. I think that we were dumping the full Produce request and Fetch response when `verbose` was `true`. Should we just remove it?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +

Review comment:
       `IllegaleStateException` sounds more appropriate here and below.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))

Review comment:
       `softwareName` -> `softwareVersion`?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>

Review comment:
       nit: Missing space before `=>`.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null)
+          partitionNode.set("records", NullNode.instance)
+        else
+          partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))

Review comment:
       This is not consistent with the changes that have been made in the automated protocol.

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       I am not entirely convinced by this change. The aim of the auto-generated protocol is to serialize/deserialise the requests/responses. Thus, it is a little weird that we only retain the size when serializing records to JSON and we actually expect bytes when deserializing from JSON. We are not consistent.
   
   I can think of the following alternatives:
   1) We keep exceptions for both the produce request and the fetch response in `RequestConvertToJson` where we could add a `recordsSizeInBytes` field instead of the `records` field.
   2) We could post-process the JSON tree where we could discard `records` and add `recordsSizeInBytes`.
   3) We could extend the autogenerated protocol to do only retain the size based on a flag. I am thinking about a `verbose` flag like we had before.
   
   I lean towards 3) at the moment. By default, we should keep the current behavior and only retains the size when `verbose` is `false`. We may find a better name than `verbose` though.
   
   @anatasiavela @lbradstreet Would this make sense?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r522472868



##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testAllResponseTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      val struct = ApiMessageType.fromApiKey(key.id).newResponse().toStruct(version)
+      val res = AbstractResponse.parseResponse(key, struct, version)
+      try {
+        RequestConvertToJson.response(res, version, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled response keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochRequestNode(): Unit = {
+    val partitionDataMap = new util.HashMap[TopicPartition, PartitionData]
+    partitionDataMap.put(new TopicPartition("topic1", 0), new PartitionData(Optional.of(0),  1))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val request = OffsetsForLeaderEpochRequest.Builder.forConsumer(partitionDataMap).build(version)
+    val actualNode = RequestConvertToJson.request(request, true)
+
+    val requestData = OffsetForLeaderEpochRequestDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochRequestDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceRequestNode(): Unit = {
+    val produceDataMap = new util.HashMap[TopicPartition, MemoryRecords]
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val serializeRecords: Boolean = false;
+    val request = ProduceRequest.Builder.forMagic(2, 0.toShort, 0, produceDataMap, "").build()
+    val actualNode = RequestConvertToJson.request(request, serializeRecords)
+
+    val requestData = new ProduceRequestData()
+    val expectedNode = ProduceRequestDataJsonConverter.write(requestData, version, serializeRecords)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochResponseNode(): Unit = {
+    val endOffsetMap = new util.HashMap[TopicPartition, EpochEndOffset]
+    endOffsetMap.put(new TopicPartition("topic1", 0), new EpochEndOffset(1, 10L))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val response = new OffsetsForLeaderEpochResponse(endOffsetMap)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = OffsetForLeaderEpochResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceResponseNode(): Unit = {
+    val responseData = new util.HashMap[TopicPartition, ProduceResponse.PartitionResponse]
+    val partResponse = new ProduceResponse.PartitionResponse(Errors.NONE, 10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonList(new ProduceResponse.RecordError(3, "Record error")), "Produce failed")
+    responseData.put(new TopicPartition("topic1", 0), partResponse)
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val response = new ProduceResponse(responseData)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = ProduceResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = ProduceResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test def testFieldsRequestDescMetrics(): Unit = {
+    val expectedFields = Set("requestHeader", "request", "response", "connection",
+      "totalTimeMs", "requestQueueTimeMs", "localTimeMs", "remoteTimeMs", "throttleTimeMs",
+      "responseQueueTimeMs", "sendTimeMs", "securityProtocol", "principal", "listener",
+      "clientInformation", "softwareName", "softwareVersion", "temporaryMemoryBytes", "messageConversionsTime")
+
+    val req = request(new AlterIsrRequest(new AlterIsrRequestData(), 0))
+    val byteBuffer = req.body[AbstractRequest].serialize(req.header)
+    val send = new NetworkSend(req.context.connectionId, byteBuffer)
+    val headerLog = RequestConvertToJson.requestHeaderNode(req.header)
+    val res = new RequestChannel.SendResponse(req, send, Some(headerLog), None)
+
+    val node = RequestConvertToJson.requestDescMetrics(req.header, res, req.loggableRequest, req.context, req.session,
+      1, 1, 1, 1, 1, 1, 1, 1, 1).asInstanceOf[ObjectNode]
+    val foundFields = getFieldNames(node)

Review comment:
       I've modified the test to validate the values, but I'm not sure the worth in doing this especially when constructing the expected output is just copying the code for the node construction into the test, unless you were thinking of another way of constructing the node. Plus it's kind of a hassle when we try to modify `requestDescMetric` because it would need to be changed in 2 places: the code and the test.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514636997



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {

Review comment:
       Good point, I'll add it.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514619891



##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       I guess this is related with the inconsistency issue, but I tried not to make too many changes. The constructors with the `data` parameters were private in some cases, so I used the `toStruct`. But for consistency, I'll change to use `toStruct` instead.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517562348



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @dajac if I understand correctly, the current generated JSON does not print the recordSet either. Is it breaking anything to return the size rather than an empty array?
   ```
               }
               if (_object.recordSet == null) {
                   _node.set("recordSet", NullNode.instance);
               } else {
                   _node.set("recordSet", new BinaryNode(new byte[]{}));
               }
   ```
   
   That said, a verbose mode where we print out a byte array seems reasonable if we don't use it in the trace logging path by default?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514882473



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.

Review comment:
       I've change it back to reflect what the auto-generated schema produces. The reason we didn't have it match the auto-generated schema was because it copies every byte of the record into the `records` field which could be expensive. If this isn't really a problem, I'll leave it as this.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r540798413



##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -121,9 +122,16 @@ object RequestChannel extends Logging {
       }
     }
 
-    def responseString(response: AbstractResponse): Option[String] = {
+    def requestLog: Option[JsonNode] = {

Review comment:
       We must use a `val` to store the `requestLog` in order to guarantee that the JSON representation is computed before the request is processed by the api layer. Otherwise, we still have a ProduceRequest without the data. The `val` may be private as well and I would put it right below `bodyAndSize` with a comment which explains why we do this.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, ObjectNode, TextNode}
+import kafka.network.RequestChannel.Session
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.requests._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => OffsetForLeaderEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ProduceRequest => ProduceRequestDataJsonConverter.write(req.data, request.version)

Review comment:
       Shouldn't we pass `false` here in order to not serialize the records?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, ObjectNode, TextNode}
+import kafka.network.RequestChannel.Session
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.requests._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => OffsetForLeaderEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ProduceRequest => ProduceRequestDataJsonConverter.write(req.data, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => OffsetForLeaderEpochResponseDataJsonConverter.write(res.data(), version)
+      case res: ProduceResponse => ProduceResponseDataJsonConverter.write(res.data(), version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data, header.headerVersion, false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestContextNode(context: RequestContext): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, requestNode: Option[JsonNode], envelope: Option[RequestChannel.Request] = None): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("isForwarded", if (envelope.isDefined) BooleanNode.TRUE else BooleanNode.FALSE)
+    node.set("forwardedRequest", envelope.map(request => requestContextNode(request.context)).getOrElse(new TextNode("")))

Review comment:
       I am not sure that this is necessary. Knowing that the request is forwarded is enough, I suppose.

##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -137,11 +145,8 @@ object RequestChannel extends Logging {
       }
     }
 
-    def requestDesc(details: Boolean): String = {
-      val forwardDescription = envelope.map { request =>
-        s"Forwarded request: ${request.context} "
-      }.getOrElse("")
-      s"$forwardDescription$header -- ${loggableRequest.toString(details)}"
+    def requestDesc: JsonNode = {

Review comment:
       It seems that this one is used from `KafkaApis` and others. It may be better to keep the original implementation for these cases.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, ObjectNode, TextNode}
+import kafka.network.RequestChannel.Session
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.requests._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => OffsetForLeaderEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ProduceRequest => ProduceRequestDataJsonConverter.write(req.data, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => OffsetForLeaderEpochResponseDataJsonConverter.write(res.data(), version)
+      case res: ProduceResponse => ProduceResponseDataJsonConverter.write(res.data(), version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data, header.headerVersion, false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestContextNode(context: RequestContext): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, requestNode: Option[JsonNode], envelope: Option[RequestChannel.Request] = None): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("isForwarded", if (envelope.isDefined) BooleanNode.TRUE else BooleanNode.FALSE)
+    node.set("forwardedRequest", envelope.map(request => requestContextNode(request.context)).getOrElse(new TextNode("")))
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", requestNode.getOrElse(new TextNode("")))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion))
+    node
+  }
+
+  def topicPartitionNode(topicPartition: TopicPartition): JsonNode = {

Review comment:
       It seems that we don't use this one any more.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{BooleanNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, ObjectNode, TextNode}
+import kafka.network.RequestChannel.Session
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.requests._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: EnvelopeRequest => EnvelopeRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => OffsetForLeaderEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ProduceRequest => ProduceRequestDataJsonConverter.write(req.data, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => OffsetForLeaderEpochResponseDataJsonConverter.write(res.data(), version)
+      case res: ProduceResponse => ProduceResponseDataJsonConverter.write(res.data(), version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data, header.headerVersion, false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestContextNode(context: RequestContext): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, requestNode: Option[JsonNode], envelope: Option[RequestChannel.Request] = None): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("isForwarded", if (envelope.isDefined) BooleanNode.TRUE else BooleanNode.FALSE)
+    node.set("forwardedRequest", envelope.map(request => requestContextNode(request.context)).getOrElse(new TextNode("")))
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", requestNode.getOrElse(new TextNode("")))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion))
+    node
+  }
+
+  def topicPartitionNode(topicPartition: TopicPartition): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("topic", new TextNode(topicPartition.topic))
+    node.set("partition", new IntNode(topicPartition.partition))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, requestNode: Option[JsonNode], responseNode: Option[JsonNode],

Review comment:
       We must pass either the `envelope ` or `isForwarded` here as well. This is the main request log and, as it is, it does not tell if the request is forwarded or not.

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -377,11 +387,24 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
                 buffer.printf("%s;%n", target.assignmentStatement(
                     String.format("new BinaryNode(Arrays.copyOf(%s, %s.length))",
-                        target.sourceVariable(), target.sourceVariable())));
+                            target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
             headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            // KIP-673: When logging requests/responses, we do not serialize the record, instead we
+            // output its sizeInBytes, because outputting the bytes is not very useful and can be
+            // quite expensive. Otherwise, we will serialize the record.
+            buffer.printf("if (_serializeRecords) {%n");
+            buffer.incrementIndent();
             buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            buffer.decrementIndent();
+            buffer.printf("} else {%n");
+            buffer.incrementIndent();
+            buffer.printf("%s;%n", target.assignmentStatement(

Review comment:
       I wonder if we should name the field `%sSizeInBytes`. I just looked at the result and having `"records":83` in the request log is not super clear to me.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r521273910



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -375,13 +385,36 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable())));
             } else {
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
+                headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+                buffer.printf("if (_serializeRecords) {%n");

Review comment:
       Understood. I think that we should revert this. I think that it makes sense to wait until we complete the migration of the remaining requests. We should have them pretty soon now.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r520807921



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -375,13 +385,36 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable())));
             } else {
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
+                headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+                buffer.printf("if (_serializeRecords) {%n");

Review comment:
       Yea, this change is because ProduceRequest uses `bytes` type. It would be worthwhile if we waited until we migrated the produce request to the automated protocol so we wouldn't have to change this.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514619891



##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       I guess this is related with the consistency issue, but I tried not to make too many changes. The constructors with the `data` parameters were private in some cases, so I used the `toStruct`. But for consistency, I'll change to use one or the other rather than alternate between the two.

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util.HashMap
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.ApiKeys
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  def createRequestsFromApiKey(apiKey: ApiKeys, version: Short): AbstractRequest = apiKey match {
+    case ApiKeys.PRODUCE => ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new HashMap[TopicPartition, MemoryRecords]()).build()
+    case ApiKeys.FETCH => new FetchRequest(new FetchRequestData(), version)
+    case ApiKeys.LIST_OFFSETS => new ListOffsetRequest(new ListOffsetRequestData().toStruct(version), version)
+    case ApiKeys.METADATA => new MetadataRequest(new MetadataRequestData(), version)
+    case ApiKeys.OFFSET_COMMIT => new OffsetCommitRequest(new OffsetCommitRequestData(), version)
+    case ApiKeys.OFFSET_FETCH => new OffsetFetchRequest(new OffsetFetchRequestData().toStruct(version), version)
+    case ApiKeys.FIND_COORDINATOR => new FindCoordinatorRequest(new FindCoordinatorRequestData().toStruct(version), version)
+    case ApiKeys.JOIN_GROUP => new JoinGroupRequest(new JoinGroupRequestData(), version)
+    case ApiKeys.HEARTBEAT => new HeartbeatRequest(new HeartbeatRequestData().toStruct(version), version)
+    case ApiKeys.LEAVE_GROUP => new LeaveGroupRequest(new LeaveGroupRequestData().toStruct(version), version)

Review comment:
       I guess this is related with the inconsistency issue, but I tried not to make too many changes. The constructors with the `data` parameters were private in some cases, so I used the `toStruct`. But for consistency, I'll change to use one or the other rather than alternate between the two.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518187139



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @lbradstreet No, it does not break anything but it is inconsistent. If we put our case aside, the purpose of the JSON converters is to convert to/from JSON so it is fair to expect that we have to maintain this contract by default. Changing it to return the size would break this contract.
   
   Regarding `_node.set("recordSet", new BinaryNode(new byte[]{}));`, I do believe that this is a bug actually...
   
   > That said, implementing a verbose mode (3) where we print out a byte array seems reasonable if we don't use it in the trace logging path by default?
   
   Agreed. When logging requests/responses, we should stick to the non-verbose mode.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518363490



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null)
+          partitionNode.set("records", NullNode.instance)
+        else
+          partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))

Review comment:
       This ends up breaking `testFormatOfProduceRequestNode()` because it can't deserialize and serialize the data because of the inconsistency problem discussed below.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514610499



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))

Review comment:
       So I'll keep `requestApiKey` as it originally was and just add a new one `requestApiKeyName`.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514636692



##########
File path: core/src/main/scala/kafka/network/RequestChannel.scala
##########
@@ -162,7 +161,7 @@ object RequestChannel extends Logging {
       }
     }
 
-    trace(s"Processor $processor received request: ${requestDesc(true)}")
+    trace(s"Processor $processor received request: ${RequestConvertToJson.requestDesc(header, loggableRequest, true).toString}")

Review comment:
       The macro does check `isTraceEnabled`, and this check is done before the string is evaluated, so it wouldn't be unnecessarily computed. But I'm also fine with gating it.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514614216



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }

Review comment:
       There was an issue before where a request type wasn't handled and it threw a `MatchError`. I added the system test to make sure all the request types were handled, but in case this happens again, I'll add the exception for unknown cases.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r520340843



##########
File path: clients/src/test/java/org/apache/kafka/common/message/SimpleExampleMessageTest.java
##########
@@ -394,7 +394,7 @@ private void testRoundTrip(SimpleExampleMessageData message,
         assertEquals(message.hashCode(), messageFromStruct.hashCode());
 
         // Check JSON serialization
-        JsonNode serializedJson = SimpleExampleMessageDataJsonConverter.write(message, version);
+        JsonNode serializedJson = SimpleExampleMessageDataJsonConverter.write(message, version, true);

Review comment:
       It seems that we can revert this back as we have the overloaded method now.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       It seems that we can remove `verbose` as we don't really use it anymore. Instead, we could set it to false for the produce request below.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version, verbose)
+      case req: ProduceRequest => produceRequestNode(req, request.version, verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short, verbose: Boolean): JsonNode = {

Review comment:
       Same here. We can remove `verbose` and set it to false for the fetch response.

##########
File path: core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala
##########
@@ -93,10 +94,8 @@ class TestRaftRequestHandler(
     val response = responseOpt match {
       case Some(response) =>
         val responseSend = request.context.buildResponse(response)
-        val responseString =
-          if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion))
-          else None
-        new RequestChannel.SendResponse(request, responseSend, responseString, None)
+        val headerLog = RequestConvertToJson.requestHeaderNode(request.header)
+        new RequestChannel.SendResponse(request, responseSend, Some(headerLog), None)

Review comment:
       I suppose that we need to keep checking `if (RequestChannel.isRequestLoggingEnabled)` here.

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testAllResponseTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      val struct = ApiMessageType.fromApiKey(key.id).newResponse().toStruct(version)
+      val res = AbstractResponse.parseResponse(key, struct, version)
+      try {
+        RequestConvertToJson.response(res, version, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled response keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochRequestNode(): Unit = {
+    val partitionDataMap = new util.HashMap[TopicPartition, PartitionData]
+    partitionDataMap.put(new TopicPartition("topic1", 0), new PartitionData(Optional.of(0),  1))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val request = OffsetsForLeaderEpochRequest.Builder.forConsumer(partitionDataMap).build(version)
+    val actualNode = RequestConvertToJson.request(request, true)
+
+    val requestData = OffsetForLeaderEpochRequestDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochRequestDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceRequestNode(): Unit = {
+    val produceDataMap = new util.HashMap[TopicPartition, MemoryRecords]
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val serializeRecords: Boolean = false;
+    val request = ProduceRequest.Builder.forMagic(2, 0.toShort, 0, produceDataMap, "").build()
+    val actualNode = RequestConvertToJson.request(request, serializeRecords)
+
+    val requestData = new ProduceRequestData()
+    val expectedNode = ProduceRequestDataJsonConverter.write(requestData, version, serializeRecords)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochResponseNode(): Unit = {
+    val endOffsetMap = new util.HashMap[TopicPartition, EpochEndOffset]
+    endOffsetMap.put(new TopicPartition("topic1", 0), new EpochEndOffset(1, 10L))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val response = new OffsetsForLeaderEpochResponse(endOffsetMap)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = OffsetForLeaderEpochResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceResponseNode(): Unit = {
+    val responseData = new util.HashMap[TopicPartition, ProduceResponse.PartitionResponse]
+    val partResponse = new ProduceResponse.PartitionResponse(Errors.NONE, 10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonList(new ProduceResponse.RecordError(3, "Record error")), "Produce failed")
+    responseData.put(new TopicPartition("topic1", 0), partResponse)
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val response = new ProduceResponse(responseData)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = ProduceResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = ProduceResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test def testFieldsRequestDescMetrics(): Unit = {
+    val expectedFields = Set("requestHeader", "request", "response", "connection",
+      "totalTimeMs", "requestQueueTimeMs", "localTimeMs", "remoteTimeMs", "throttleTimeMs",
+      "responseQueueTimeMs", "sendTimeMs", "securityProtocol", "principal", "listener",
+      "clientInformation", "softwareName", "softwareVersion", "temporaryMemoryBytes", "messageConversionsTime")
+
+    val req = request(new AlterIsrRequest(new AlterIsrRequestData(), 0))
+    val byteBuffer = req.body[AbstractRequest].serialize(req.header)
+    val send = new NetworkSend(req.context.connectionId, byteBuffer)
+    val headerLog = RequestConvertToJson.requestHeaderNode(req.header)
+    val res = new RequestChannel.SendResponse(req, send, Some(headerLog), None)
+
+    val node = RequestConvertToJson.requestDescMetrics(req.header, res, req.loggableRequest, req.context, req.session,
+      1, 1, 1, 1, 1, 1, 1, 1, 1).asInstanceOf[ObjectNode]
+    val foundFields = getFieldNames(node)

Review comment:
       Thinking a bit more about this. I think that it may be worth validating the values as well to ensure that the input maps to the output. If we use different values for each field, we could verify that the output JSON object is correct. `JSONNode` are comparable so we could construct the expected output and use it in `assertEquals`. What do you think?

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testAllResponseTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      val struct = ApiMessageType.fromApiKey(key.id).newResponse().toStruct(version)
+      val res = AbstractResponse.parseResponse(key, struct, version)
+      try {
+        RequestConvertToJson.response(res, version, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled response keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochRequestNode(): Unit = {
+    val partitionDataMap = new util.HashMap[TopicPartition, PartitionData]
+    partitionDataMap.put(new TopicPartition("topic1", 0), new PartitionData(Optional.of(0),  1))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val request = OffsetsForLeaderEpochRequest.Builder.forConsumer(partitionDataMap).build(version)
+    val actualNode = RequestConvertToJson.request(request, true)
+
+    val requestData = OffsetForLeaderEpochRequestDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochRequestDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceRequestNode(): Unit = {
+    val produceDataMap = new util.HashMap[TopicPartition, MemoryRecords]
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val serializeRecords: Boolean = false;
+    val request = ProduceRequest.Builder.forMagic(2, 0.toShort, 0, produceDataMap, "").build()
+    val actualNode = RequestConvertToJson.request(request, serializeRecords)
+
+    val requestData = new ProduceRequestData()
+    val expectedNode = ProduceRequestDataJsonConverter.write(requestData, version, serializeRecords)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochResponseNode(): Unit = {
+    val endOffsetMap = new util.HashMap[TopicPartition, EpochEndOffset]
+    endOffsetMap.put(new TopicPartition("topic1", 0), new EpochEndOffset(1, 10L))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val response = new OffsetsForLeaderEpochResponse(endOffsetMap)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = OffsetForLeaderEpochResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceResponseNode(): Unit = {
+    val responseData = new util.HashMap[TopicPartition, ProduceResponse.PartitionResponse]
+    val partResponse = new ProduceResponse.PartitionResponse(Errors.NONE, 10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonList(new ProduceResponse.RecordError(3, "Record error")), "Produce failed")
+    responseData.put(new TopicPartition("topic1", 0), partResponse)
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val response = new ProduceResponse(responseData)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = ProduceResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = ProduceResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test def testFieldsRequestDescMetrics(): Unit = {
+    val expectedFields = Set("requestHeader", "request", "response", "connection",
+      "totalTimeMs", "requestQueueTimeMs", "localTimeMs", "remoteTimeMs", "throttleTimeMs",
+      "responseQueueTimeMs", "sendTimeMs", "securityProtocol", "principal", "listener",
+      "clientInformation", "softwareName", "softwareVersion", "temporaryMemoryBytes", "messageConversionsTime")
+
+    val req = request(new AlterIsrRequest(new AlterIsrRequestData(), 0))
+    val byteBuffer = req.body[AbstractRequest].serialize(req.header)
+    val send = new NetworkSend(req.context.connectionId, byteBuffer)
+    val headerLog = RequestConvertToJson.requestHeaderNode(req.header)
+    val res = new RequestChannel.SendResponse(req, send, Some(headerLog), None)
+
+    val node = RequestConvertToJson.requestDescMetrics(req.header, res, req.loggableRequest, req.context, req.session,
+      1, 1, 1, 1, 1, 1, 1, 1, 1).asInstanceOf[ObjectNode]
+    val foundFields = getFieldNames(node)
+
+    assertEquals(expectedFields, foundFields)
+  }
+
+  def request(req: AbstractRequest): RequestChannel.Request = {

Review comment:
       nit: Could we make it private?

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testAllResponseTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      val struct = ApiMessageType.fromApiKey(key.id).newResponse().toStruct(version)
+      val res = AbstractResponse.parseResponse(key, struct, version)
+      try {
+        RequestConvertToJson.response(res, version, false)
+      } catch {
+        case _ : IllegalStateException => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled response keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochRequestNode(): Unit = {
+    val partitionDataMap = new util.HashMap[TopicPartition, PartitionData]
+    partitionDataMap.put(new TopicPartition("topic1", 0), new PartitionData(Optional.of(0),  1))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val request = OffsetsForLeaderEpochRequest.Builder.forConsumer(partitionDataMap).build(version)
+    val actualNode = RequestConvertToJson.request(request, true)
+
+    val requestData = OffsetForLeaderEpochRequestDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochRequestDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceRequestNode(): Unit = {
+    val produceDataMap = new util.HashMap[TopicPartition, MemoryRecords]
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val serializeRecords: Boolean = false;
+    val request = ProduceRequest.Builder.forMagic(2, 0.toShort, 0, produceDataMap, "").build()
+    val actualNode = RequestConvertToJson.request(request, serializeRecords)
+
+    val requestData = new ProduceRequestData()
+    val expectedNode = ProduceRequestDataJsonConverter.write(requestData, version, serializeRecords)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfOffsetsForLeaderEpochResponseNode(): Unit = {
+    val endOffsetMap = new util.HashMap[TopicPartition, EpochEndOffset]
+    endOffsetMap.put(new TopicPartition("topic1", 0), new EpochEndOffset(1, 10L))
+
+    val version: Short = ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion
+    val response = new OffsetsForLeaderEpochResponse(endOffsetMap)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = OffsetForLeaderEpochResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = OffsetForLeaderEpochResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test
+  def testFormatOfProduceResponseNode(): Unit = {
+    val responseData = new util.HashMap[TopicPartition, ProduceResponse.PartitionResponse]
+    val partResponse = new ProduceResponse.PartitionResponse(Errors.NONE, 10000, RecordBatch.NO_TIMESTAMP, 100, Collections.singletonList(new ProduceResponse.RecordError(3, "Record error")), "Produce failed")
+    responseData.put(new TopicPartition("topic1", 0), partResponse)
+
+    val version: Short = ApiKeys.PRODUCE.latestVersion
+    val response = new ProduceResponse(responseData)
+    val actualNode = RequestConvertToJson.response(response, version, true)
+
+    val requestData = ProduceResponseDataJsonConverter.read(actualNode, version)
+    val expectedNode = ProduceResponseDataJsonConverter.write(requestData, version, true)
+
+    assertEquals(expectedNode, actualNode)
+  }
+
+  @Test def testFieldsRequestDescMetrics(): Unit = {

Review comment:
       nit: Could we add a new line after `@Test`?

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = key.latestVersion()
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }

Review comment:
       nit: We would usually use a `val` here and write the block as follow:
   ```
   val req = if (key == ApiKeys.PRODUCE) {
     ...
   } else {
     ...
   }
   ```

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.net.InetAddress
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import kafka.network
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.memory.MemoryPool
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.{ClientInformation, ListenerName, NetworkSend}
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{MemoryRecords, RecordBatch}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol}
+import org.easymock.EasyMock.createNiceMock
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {

Review comment:
       nit: We tend to write this as follow:
   ```
   ApiKeys.values().foreach { key => 
     ...
   }
   ```
   This is another similar case in `testAllResponseTypesHandled`.

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -375,13 +385,36 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable())));
             } else {
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
+                headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+                buffer.printf("if (_serializeRecords) {%n");

Review comment:
       I am not sure to understand why we are doing this change now. It was not there before. Could you elaborate?
   
   Is it because the produce request does use `bytes` as type? It seems `bytes` was used there by mistake. FYI, the PR which will migrate the produce request to the automated protocol will change it to `records`: https://github.com/apache/kafka/pull/9401/files#diff-e6dde0832c4873b546db2ad0d37be899855a436972b0a2c5008b100a0ffff64dR50.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518384421



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version, verbose)
+      case req: ProduceRequest => produceRequestNode(req, request.version, verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short, verbose: Boolean): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version, verbose)
+      case res: ProduceResponse => produceResponseNode(res, version, verbose)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader, verbose: Boolean): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), verbose).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,

Review comment:
       Yeah, I was thinking about a basic test like this which verifies the fields. We had few typos so it is good to test this.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r541240558



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -377,11 +387,24 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                 headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS);
                 buffer.printf("%s;%n", target.assignmentStatement(
                     String.format("new BinaryNode(Arrays.copyOf(%s, %s.length))",
-                        target.sourceVariable(), target.sourceVariable())));
+                            target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
             headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            // KIP-673: When logging requests/responses, we do not serialize the record, instead we
+            // output its sizeInBytes, because outputting the bytes is not very useful and can be
+            // quite expensive. Otherwise, we will serialize the record.
+            buffer.printf("if (_serializeRecords) {%n");
+            buffer.incrementIndent();
             buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            buffer.decrementIndent();
+            buffer.printf("} else {%n");
+            buffer.incrementIndent();
+            buffer.printf("%s;%n", target.assignmentStatement(

Review comment:
       I do see the issue of it not being super clear, but I don't think we can change the field name from here. Doing `%sSizeInBytes` would just add the name at the end of the line which would result in a compilation error. Unless you mean to change the field name in the `.json` file, but it would change the name for both the serialize and non-serialize case.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517562348



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @dajac if I understand correctly, the current generated JSON does not print the recordSet either. Is it breaking anything to return the size rather than an empty array?
   ```
               }
               if (_object.recordSet == null) {
                   _node.set("recordSet", NullNode.instance);
               } else {
                   _node.set("recordSet", new BinaryNode(new byte[]{}));
               }
   ```
   
   That said, implementing a verbose mode (3) where we print out a byte array seems reasonable if we don't use it in the trace logging path by default?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517665383



##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +380,9 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
-            headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
-            buffer.printf("%s;%n", target.assignmentStatement("new BinaryNode(new byte[]{})"));
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
+            buffer.printf("%s;%n", target.assignmentStatement(
+                    String.format("new IntNode(%s.sizeInBytes())", target.sourceVariable())));

Review comment:
       @lbradstreet The current generated JSON does not print the recordSet either. But when we are serializing a BinaryNode with an empty array, it is still being deserialized as a BinaryNode, so it doesn't break anything.
   ```
               buffer.printf("%s;%n", target.assignmentStatement(
                   String.format("MemoryRecords.readableRecords(ByteBuffer.wrap(MessageUtil.jsonNodeToBinary(%s, \"%s\")))",
                       target.sourceVariable(), target.humanReadableName())));
   ```
   So I believe the concern is that I had proposed to change the serialization to an IntNode, but the deserialization still expects a BinaryNode which I should have also changed.
   Another alternative I could fix this by changing the deserialization to expect an IntNode, but I'm for adding a verbose flag
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] lbradstreet commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

lbradstreet commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r526255450



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version, false)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = requestDesc(header, req).asInstanceOf[ObjectNode]
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, serializeRecords: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null) {
+          partitionNode.set("records", NullNode.instance)
+        } else {
+          if (serializeRecords)
+            partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))
+          else
+            partitionNode.set("records", new IntNode(partitionData.validBytes()))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochResponseNode(response: OffsetsForLeaderEpochResponse, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 2) {
+      node.set("throttleTimeMs", new IntNode(response.throttleTimeMs))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(response.responses)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("errorCode", new ShortNode(partitionData.error.code))
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 1) partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionNode.set("endOffset", new LongNode(partitionData.endOffset))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceResponseNode(response: ProduceResponse, version: Short): JsonNode = {

Review comment:
       Seeing as https://github.com/apache/kafka/pull/9401 is about to merge, maybe we can wait for that before merging this and drop our manual JSON generation code for produce request/response?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r526511067



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version, false)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), false).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = requestDesc(header, req).asInstanceOf[ObjectNode]
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, serializeRecords: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null) {
+          partitionNode.set("records", NullNode.instance)
+        } else {
+          if (serializeRecords)
+            partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))
+          else
+            partitionNode.set("records", new IntNode(partitionData.validBytes()))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochResponseNode(response: OffsetsForLeaderEpochResponse, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 2) {
+      node.set("throttleTimeMs", new IntNode(response.throttleTimeMs))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(response.responses)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("errorCode", new ShortNode(partitionData.error.code))
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 1) partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionNode.set("endOffset", new LongNode(partitionData.endOffset))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceResponseNode(response: ProduceResponse, version: Short): JsonNode = {

Review comment:
       Yes! we're waiting on #9401 and #9547 to merge. Once they're merged, I'll pick this PR back up and will remove all the temporary case handling done for the Produce and OffsetsForLeaderEpoch request/response.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518387030



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null)
+          partitionNode.set("records", NullNode.instance)
+        else
+          partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))

Review comment:
       Right. We need to update that test for the time being until we migrate the produce request/response. We may be able to verify the output differently.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r517645718



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       Yes, I kept it because `requestDescMetrics` originally had the `detailsEnabled`, but I'll remove it since none of the JsonConverters differentiates between verbose and non-verbose.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#issuecomment-730292123


   @anatasiavela Both PRs have been merged so we can proceed with this one.
   
   There is something that we must consider that I was not aware of: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java#L235. When the ProduceRequest is processed in the KafkaApis layer, its internal data is set to null to free up the memory. That means that we won't have it to log the request. We need to take this into account.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514621561



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))

Review comment:
       I'm for it, that would be more descriptive.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] dajac commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

dajac commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r518187681



##########
File path: clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
##########
@@ -346,6 +346,10 @@ public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) {
         return partitionSizes.keySet();
     }
 
+    public Map<TopicPartition, Integer> partitionSizes() {
+        return partitionSizes;
+    }

Review comment:
       Is this one still used? It seems that it is not the case.

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -227,9 +227,13 @@ private void generateVariableLengthTargetFromJson(Target target, Versions curVer
             headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS);
             headerGenerator.addImport(MessageGenerator.BYTE_BUFFER_CLASS);
             headerGenerator.addImport(MessageGenerator.MEMORY_RECORDS_CLASS);
+            buffer.printf("if (_verbose) {%n");
+            buffer.incrementIndent();

Review comment:
       I think that we can safely assume that when a request/response is serialized with `verbose` equals to `false`, we are not going to deserialize it. Therefore, I suggest to drop the handling of `verbose` on the read path.

##########
File path: core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala
##########
@@ -39,7 +40,7 @@ class TestRaftRequestHandler(
 
   override def handle(request: RequestChannel.Request): Unit = {
     try {
-      trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" +
+      trace(s"Handling request:${RequestConvertToJson.requestDesc(request.header, request.loggableRequest, true)} from connection ${request.context.connectionId};" +

Review comment:
       `verbose` should be `false` here (if we keep it).

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -261,7 +265,7 @@ private void generateWrite(String className,
                                StructSpec struct,
                                Versions parentVersions) {
         headerGenerator.addImport(MessageGenerator.JSON_NODE_CLASS);
-        buffer.printf("public static JsonNode write(%s _object, short _version) {%n",
+        buffer.printf("public static JsonNode write(%s _object, short _version, boolean _verbose) {%n",

Review comment:
       Instead of changing the usage of this method everywhere in the code base, how about generating an overloaded method which call this one with `verbose=true`? I only expect this one to be used by the request logger at the moment so it is also more convenient.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version, verbose)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version, verbose)
+      case req: ProduceRequest => produceRequestNode(req, request.version, verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short, verbose: Boolean): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version, verbose)
+      case res: ProduceResponse => produceResponseNode(res, version, verbose)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version, verbose)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version, verbose)
+      case _ => throw new IllegalStateException(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader, verbose: Boolean): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion(), verbose).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareVersion", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,

Review comment:
       Could we add a unit test for this method?

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +384,18 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
             headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
+            buffer.printf("if (_verbose) {%n");

Review comment:
       I wonder if we could find a better name than `verbose`. Perhaps, we could be more explicit and use something like `serializeRecords` as we actually only use it for this at the moment. What do you think?

##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -3354,7 +3354,7 @@ class KafkaApis(val requestChannel: RequestChannel,
       case Some(response) =>
         val responseSend = request.context.buildResponse(response)
         val responseString =
-          if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion))
+          if (RequestChannel.isRequestLoggingEnabled) Some(RequestConvertToJson.response(response, request.context.apiVersion, true))

Review comment:
       I think that we should set this to `false`.

##########
File path: core/src/main/scala/kafka/server/KafkaApis.scala
##########
@@ -131,7 +131,7 @@ class KafkaApis(val requestChannel: RequestChannel,
    */
   override def handle(request: RequestChannel.Request): Unit = {
     try {
-      trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" +
+      trace(s"Handling request:${RequestConvertToJson.requestDesc(request.header, request.loggableRequest, true).toString} from connection ${request.context.connectionId};" +

Review comment:
       `verbose` should be `false` here (if we keep it).

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, TimestampType}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = 0
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]()).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : AssertionError => unhandledKeys += key.toString

Review comment:
       `IllegalStateException`?

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {

Review comment:
       I think that we can remove this `verbose` flag here and only set it to false for both the produce request and the fetch response. I don't think that we will ever want to print out the bytes in the request log. Previously, we were printing out different things based on the flag but we never printed out the bytes: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java#L314.

##########
File path: core/src/test/scala/unit/kafka/network/RequestConvertToJsonTest.scala
##########
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.nio.ByteBuffer
+import java.util
+import java.util.{Collections, Optional}
+
+import org.apache.kafka.common.TopicPartition
+import org.apache.kafka.common.message._
+import org.junit.Test
+import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, TimestampType}
+import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest.PartitionData
+import org.apache.kafka.common.requests._
+import org.junit.Assert.assertEquals
+
+import scala.collection.mutable.ArrayBuffer
+
+class RequestConvertToJsonTest {
+
+  @Test
+  def testAllRequestTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = 0
+      var req: AbstractRequest = null
+      if (key == ApiKeys.PRODUCE) {
+        // There's inconsistency with the toStruct schema in ProduceRequest
+        // and ProduceRequestDataJsonConverters where the field names don't
+        // match so the struct does not have the correct field names. This is
+        // a temporary workaround until ProduceRequest starts using ProduceRequestData
+        req = ProduceRequest.Builder.forCurrentMagic(0.toShort, 10000, new util.HashMap[TopicPartition, MemoryRecords]()).build()
+      } else {
+        val struct = ApiMessageType.fromApiKey(key.id).newRequest().toStruct(version)
+        req = AbstractRequest.parseRequest(key, version, struct)
+      }
+      try {
+        RequestConvertToJson.request(req, false)
+      } catch {
+        case _ : AssertionError => unhandledKeys += key.toString
+      }
+    })
+    assertEquals("Unhandled request keys", ArrayBuffer.empty, unhandledKeys)
+  }
+
+  @Test
+  def testAllResponseTypesHandled(): Unit = {
+    val unhandledKeys = ArrayBuffer[String]()
+    ApiKeys.values().foreach(key => {
+      val version: Short = 0
+      val struct = ApiMessageType.fromApiKey(key.id).newResponse().toStruct(version)
+      val res = AbstractResponse.parseResponse(key, struct, version)
+      try {
+        RequestConvertToJson.response(res, version, false)
+      } catch {
+        case _ : AssertionError => unhandledKeys += key.toString

Review comment:
       `IllegalStateException`?

##########
File path: generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
##########
@@ -380,8 +384,18 @@ private void generateVariableLengthTargetToJson(Target target, Versions versions
                         target.sourceVariable(), target.sourceVariable())));
             }
         } else if (target.field().type().isRecords()) {
+            headerGenerator.addImport(MessageGenerator.INT_NODE_CLASS);
             headerGenerator.addImport(MessageGenerator.BINARY_NODE_CLASS);
+            buffer.printf("if (_verbose) {%n");

Review comment:
       Could we also add a comment here which explains why we are doing this? We can also add the KIP number.

##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import java.util
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, BinaryNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.network.ClientInformation
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data, request.version)
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data, request.version)
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version)
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data, request.version)
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data, request.version)
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version)
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data, request.version)
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data, request.version)
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data, request.version)
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data, request.version)
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data, request.version)
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data, request.version)
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data, request.version)
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data, request.version)
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version)
+      case req: ProduceRequest => produceRequestNode(req, request.version)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data, request.version)
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data, request.version)
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data, request.version)
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data, request.version)
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data, request.version)
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data, request.version)
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version)
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data, request.version)
+      case _ => throw new AssertionError(s"ApiKey ${request.api} is not currently handled in `request`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data, version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data, version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data, version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data, version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data, version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data, version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data, version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data, version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data, version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data, version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data, version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data, version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data, version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data, version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+      case _ => throw new AssertionError(s"ApiKey $response is not currently handled in `response`, the " +
+        "code should be updated to do so.");
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKeyName", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def clientInfoNode(clientInfo: ClientInformation): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("softwareName", new TextNode(clientInfo.softwareName()))
+    node.set("softwareName", new TextNode(clientInfo.softwareVersion()))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                         context: RequestContext, session: Session, verbose: Boolean,
+                         totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                         apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                         responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                         messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTimeMs", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTimeMs", new DoubleNode(requestQueueTimeMs))
+    node.set("localTimeMs", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTimeMs", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTimeMs", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTimeMs", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTimeMs", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", clientInfoNode(context.clientInformation))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData) =>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        partitionsArray.add(partitionNode)
+      }
+      topicNode.set("partitions", partitionsArray)
+      topicsArray.add(topicNode)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    val topics = CollectionUtils.groupPartitionDataByTopic(request.partitionRecordsOrFail())
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    topics.forEach { (topicName, partitions) =>
+      val topicNode = new ObjectNode(JsonNodeFactory.instance)
+      topicNode.set("name", new TextNode(topicName))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      partitions.forEach { (partitionIndex, partitionData)=>
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionIndex))
+        if (partitionData == null)
+          partitionNode.set("records", NullNode.instance)
+        else
+          partitionNode.set("records", new BinaryNode(util.Arrays.copyOf(partitionData.buffer().array(), partitionData.validBytes())))

Review comment:
       Right. I think that we will change it to `records` type when we will migrate the produce request. Anyway, we should not serialize the bytes to JSON here but rather put the size. We should retain the size of the records set for both the produce request and the fetch response.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org

[GitHub] [kafka] anatasiavela commented on a change in pull request #9526: KAFKA-10525: Emit JSONs with new auto-generated schema

Posted by GitBox <gi...@apache.org>.

anatasiavela commented on a change in pull request #9526:
URL: https://github.com/apache/kafka/pull/9526#discussion_r514861991



##########
File path: core/src/main/scala/kafka/network/RequestConvertToJson.scala
##########
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka.network
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.{ArrayNode, DoubleNode, IntNode, JsonNodeFactory, LongNode, NullNode, ObjectNode, ShortNode, TextNode}
+import kafka.network.RequestChannel.{Response, Session}
+import org.apache.kafka.common.message._
+import org.apache.kafka.common.protocol.Errors
+import org.apache.kafka.common.record.RecordBatch
+import org.apache.kafka.common.requests._
+import org.apache.kafka.common.utils.CollectionUtils
+
+import scala.jdk.CollectionConverters._
+
+object RequestConvertToJson {
+  def request(request: AbstractRequest, verbose: Boolean): JsonNode = {
+    request match {
+      case req: AddOffsetsToTxnRequest => AddOffsetsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AddPartitionsToTxnRequest => AddPartitionsToTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterClientQuotasRequest => AlterClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterConfigsRequest => AlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterIsrRequest => AlterIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterPartitionReassignmentsRequest => AlterPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: AlterReplicaLogDirsRequest => AlterReplicaLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case res: AlterUserScramCredentialsRequest => AlterUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ApiVersionsRequest => ApiVersionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: BeginQuorumEpochRequest => BeginQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ControlledShutdownRequest => ControlledShutdownRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateAclsRequest => CreateAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateDelegationTokenRequest => CreateDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreatePartitionsRequest => CreatePartitionsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: CreateTopicsRequest => CreateTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteAclsRequest => DeleteAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteGroupsRequest => DeleteGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteRecordsRequest => DeleteRecordsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DeleteTopicsRequest => DeleteTopicsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeAclsRequest => DescribeAclsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeClientQuotasRequest => DescribeClientQuotasRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeConfigsRequest => DescribeConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeDelegationTokenRequest => DescribeDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeGroupsRequest => DescribeGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeLogDirsRequest => DescribeLogDirsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: DescribeQuorumRequest => DescribeQuorumRequestDataJsonConverter.write(req.data, request.version())
+      case res: DescribeUserScramCredentialsRequest => DescribeUserScramCredentialsRequestDataJsonConverter.write(res.data(), request.version())
+      case req: ElectLeadersRequest => ElectLeadersRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndTxnRequest => EndTxnRequestDataJsonConverter.write(req.data(), request.version())
+      case req: EndQuorumEpochRequest => EndQuorumEpochRequestDataJsonConverter.write(req.data, request.version())
+      case req: ExpireDelegationTokenRequest => ExpireDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FetchRequest => FetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: FindCoordinatorRequest => FindCoordinatorRequestDataJsonConverter.write(req.data(), request.version())
+      case req: HeartbeatRequest => HeartbeatRequestDataJsonConverter.write(req.data(), request.version())
+      case req: IncrementalAlterConfigsRequest => IncrementalAlterConfigsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: InitProducerIdRequest => InitProducerIdRequestDataJsonConverter.write(req.data(), request.version())
+      case req: JoinGroupRequest => JoinGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaderAndIsrRequest => LeaderAndIsrRequestDataJsonConverter.write(req.data(), request.version())
+      case req: LeaveGroupRequest => LeaveGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListGroupsRequest => ListGroupsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListOffsetRequest => ListOffsetRequestDataJsonConverter.write(req.data(), request.version())
+      case req: ListPartitionReassignmentsRequest => ListPartitionReassignmentsRequestDataJsonConverter.write(req.data(), request.version())
+      case req: MetadataRequest => MetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetCommitRequest => OffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetDeleteRequest => OffsetDeleteRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetFetchRequest => OffsetFetchRequestDataJsonConverter.write(req.data(), request.version())
+      case req: OffsetsForLeaderEpochRequest => offsetsForLeaderEpochRequestNode(req, request.version())
+      case req: ProduceRequest => produceRequestNode(req, request.version(), verbose)
+      case req: RenewDelegationTokenRequest => RenewDelegationTokenRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslAuthenticateRequest => SaslAuthenticateRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SaslHandshakeRequest => SaslHandshakeRequestDataJsonConverter.write(req.data(), request.version())
+      case req: StopReplicaRequest => StopReplicaRequestDataJsonConverter.write(req.data(), request.version())
+      case req: SyncGroupRequest => SyncGroupRequestDataJsonConverter.write(req.data(), request.version())
+      case req: TxnOffsetCommitRequest => TxnOffsetCommitRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateFeaturesRequest => UpdateFeaturesRequestDataJsonConverter.write(req.data(), request.version())
+      case req: UpdateMetadataRequest => UpdateMetadataRequestDataJsonConverter.write(req.data(), request.version())
+      case req: VoteRequest => VoteRequestDataJsonConverter.write(req.data, request.version())
+      case req: WriteTxnMarkersRequest => WriteTxnMarkersRequestDataJsonConverter.write(req.data(), request.version())
+    }
+  }
+
+  def response(response: AbstractResponse, version: Short): JsonNode = {
+    response match {
+      case res: AddOffsetsToTxnResponse => AddOffsetsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AddPartitionsToTxnResponse => AddPartitionsToTxnResponseDataJsonConverter.write(res.data, version)
+      case res: AlterClientQuotasResponse => AlterClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterConfigsResponse => AlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterIsrResponse => AlterIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterPartitionReassignmentsResponse => AlterPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterReplicaLogDirsResponse => AlterReplicaLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: AlterUserScramCredentialsResponse => AlterUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ApiVersionsResponse => ApiVersionsResponseDataJsonConverter.write(res.data, version)
+      case res: BeginQuorumEpochResponse => BeginQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ControlledShutdownResponse => ControlledShutdownResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateAclsResponse => CreateAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateDelegationTokenResponse => CreateDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: CreatePartitionsResponse => CreatePartitionsResponseDataJsonConverter.write(res.data(), version)
+      case res: CreateTopicsResponse => CreateTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteAclsResponse => DeleteAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteGroupsResponse => DeleteGroupsResponseDataJsonConverter.write(res.data, version)
+      case res: DeleteRecordsResponse => DeleteRecordsResponseDataJsonConverter.write(res.data(), version)
+      case res: DeleteTopicsResponse => DeleteTopicsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeAclsResponse => DescribeAclsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeClientQuotasResponse => DescribeClientQuotasResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeConfigsResponse => DescribeConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeDelegationTokenResponse => DescribeDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeGroupsResponse => DescribeGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeLogDirsResponse => DescribeLogDirsResponseDataJsonConverter.write(res.data(), version)
+      case res: DescribeQuorumResponse => DescribeQuorumResponseDataJsonConverter.write(res.data, version)
+      case res: DescribeUserScramCredentialsResponse => DescribeUserScramCredentialsResponseDataJsonConverter.write(res.data(), version)
+      case res: ElectLeadersResponse => ElectLeadersResponseDataJsonConverter.write(res.data(), version)
+      case res: EndTxnResponse => EndTxnResponseDataJsonConverter.write(res.data, version)
+      case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version)
+      case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data(), version)
+      case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data(), version)
+      case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data(), version)
+      case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data(), version)
+      case res: InitProducerIdResponse => InitProducerIdResponseDataJsonConverter.write(res.data, version)
+      case res: JoinGroupResponse => JoinGroupResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaderAndIsrResponse => LeaderAndIsrResponseDataJsonConverter.write(res.data(), version)
+      case res: LeaveGroupResponse => LeaveGroupResponseDataJsonConverter.write(res.data, version)
+      case res: ListGroupsResponse => ListGroupsResponseDataJsonConverter.write(res.data(), version)
+      case res: ListOffsetResponse => ListOffsetResponseDataJsonConverter.write(res.data(), version)
+      case res: ListPartitionReassignmentsResponse => ListPartitionReassignmentsResponseDataJsonConverter.write(res.data(), version)
+      case res: MetadataResponse => MetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetCommitResponse => OffsetCommitResponseDataJsonConverter.write(res.data(), version)
+      case res: OffsetDeleteResponse => OffsetDeleteResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetFetchResponse => OffsetFetchResponseDataJsonConverter.write(res.data, version)
+      case res: OffsetsForLeaderEpochResponse => offsetsForLeaderEpochResponseNode(res, version)
+      case res: ProduceResponse => produceResponseNode(res, version)
+      case res: RenewDelegationTokenResponse => RenewDelegationTokenResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslAuthenticateResponse => SaslAuthenticateResponseDataJsonConverter.write(res.data(), version)
+      case res: SaslHandshakeResponse => SaslHandshakeResponseDataJsonConverter.write(res.data(), version)
+      case res: StopReplicaResponse => StopReplicaResponseDataJsonConverter.write(res.data(), version)
+      case res: SyncGroupResponse => SyncGroupResponseDataJsonConverter.write(res.data, version)
+      case res: TxnOffsetCommitResponse => TxnOffsetCommitResponseDataJsonConverter.write(res.data, version)
+      case res: UpdateFeaturesResponse => UpdateFeaturesResponseDataJsonConverter.write(res.data(), version)
+      case res: UpdateMetadataResponse => UpdateMetadataResponseDataJsonConverter.write(res.data(), version)
+      case res: WriteTxnMarkersResponse => WriteTxnMarkersResponseDataJsonConverter.write(res.data, version)
+      case res: VoteResponse => VoteResponseDataJsonConverter.write(res.data, version)
+    }
+  }
+
+  def requestHeaderNode(header: RequestHeader): JsonNode = {
+    val node = RequestHeaderDataJsonConverter.write(header.data(), header.headerVersion()).asInstanceOf[ObjectNode]
+    node.set("requestApiKey", new TextNode(header.apiKey.toString))
+    node
+  }
+
+  def requestDescMetrics(header: RequestHeader, res: Response, req: AbstractRequest,
+                  context: RequestContext, session: Session, verbose: Boolean,
+                  totalTimeMs: Double, requestQueueTimeMs: Double, apiLocalTimeMs: Double,
+                  apiRemoteTimeMs: Double, apiThrottleTimeMs: Long, responseQueueTimeMs: Double,
+                  responseSendTimeMs: Double, temporaryMemoryBytes: Long,
+                  messageConversionsTimeMs: Double): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node.set("response", res.responseLog.getOrElse(new TextNode("")))
+    node.set("connection", new TextNode(context.connectionId))
+    node.set("totalTime", new DoubleNode(totalTimeMs))
+    node.set("requestQueueTime", new DoubleNode(requestQueueTimeMs))
+    node.set("localTime", new DoubleNode(apiLocalTimeMs))
+    node.set("remoteTime", new DoubleNode(apiRemoteTimeMs))
+    node.set("throttleTime", new LongNode(apiThrottleTimeMs))
+    node.set("responseQueueTime", new DoubleNode(responseQueueTimeMs))
+    node.set("sendTime", new DoubleNode(responseSendTimeMs))
+    node.set("securityProtocol", new TextNode(context.securityProtocol.toString))
+    node.set("principal", new TextNode(session.principal.toString))
+    node.set("listener", new TextNode(context.listenerName.value))
+    node.set("clientInformation", new TextNode(context.clientInformation.toString))
+    if (temporaryMemoryBytes > 0)
+      node.set("temporaryMemoryBytes", new LongNode(temporaryMemoryBytes))
+    if (messageConversionsTimeMs > 0)
+      node.set("messageConversionsTime", new DoubleNode(messageConversionsTimeMs))
+    node
+  }
+
+  def requestDesc(header: RequestHeader, req: AbstractRequest, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    node.set("requestHeader", requestHeaderNode(header))
+    node.set("request", request(req, verbose))
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def offsetsForLeaderEpochRequestNode(request: OffsetsForLeaderEpochRequest, version: Short): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      node.set("replicaId", new IntNode(request.replicaId))
+    }
+    val topicsToPartitionEpochs = CollectionUtils.groupPartitionDataByTopic(request.epochsByTopicPartition)
+    val topicsArray = new ArrayNode(JsonNodeFactory.instance)
+    for (topicToEpochs <- topicsToPartitionEpochs.entrySet.asScala) {
+      val topicsData = new ObjectNode(JsonNodeFactory.instance)
+      topicsData.set("name", new TextNode(topicToEpochs.getKey))
+      val partitionsArray = new ArrayNode(JsonNodeFactory.instance)
+      for (partitionEpoch <- topicToEpochs.getValue.entrySet.asScala) {
+        val partitionData = partitionEpoch.getValue
+        val partitionNode = new ObjectNode(JsonNodeFactory.instance)
+        partitionNode.set("partitionIndex", new IntNode(partitionEpoch.getKey))
+        partitionNode.set("leaderEpoch", new IntNode(partitionData.leaderEpoch))
+        if (version >= 2) {
+          val leaderEpoch = partitionData.currentLeaderEpoch
+          partitionNode.set("currentLeaderEpoch", new IntNode(leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)))
+        }
+        partitionsArray.add(partitionNode)
+      }
+      topicsData.set("partitions", partitionsArray)
+      topicsArray.add(topicsData)
+    }
+    node.set("topics", topicsArray)
+    node
+  }
+
+  /**
+   * Temporary until switch to use the generated schemas.
+   */
+  def produceRequestNode(request: ProduceRequest, version: Short, verbose: Boolean): JsonNode = {
+    val node = new ObjectNode(JsonNodeFactory.instance)
+    if (version >= 3) {
+      if (request.transactionalId == null) {
+        node.set("transactionalId", NullNode.instance)
+      } else {
+        node.set("transactionalId", new TextNode(request.transactionalId))
+      }
+    }
+    node.set("acks", new ShortNode(request.acks))
+    node.set("timeoutMs", new IntNode(request.timeout))
+    if (verbose) {
+      val partSizes = new ArrayNode(JsonNodeFactory.instance)
+      for (partSize <- request.partitionSizes().entrySet.asScala) {
+        val part = new ObjectNode(JsonNodeFactory.instance)
+        val topic = partSize.getKey
+        part.set("partition", new TextNode(topic.toString))
+        part.set("size", new IntNode(partSize.getValue))
+        partSizes.add(part)
+      }
+      node.set("partitionSizes", partSizes)
+    } else {
+      node.set("partitionSizes", new IntNode(request.partitionSizes.size))

Review comment:
       Changed to match the auto-generated schema.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org