You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jg...@apache.org on 2016/11/15 00:46:27 UTC
[1/5] kafka git commit: KAFKA-2066;
Use client-side FetchRequest/FetchResponse on server
Repository: kafka
Updated Branches:
refs/heads/trunk 1f1d45006 -> 3b4c34794
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/KafkaApis.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala
index c6c8dbd..94ae419 100644
--- a/core/src/main/scala/kafka/server/KafkaApis.scala
+++ b/core/src/main/scala/kafka/server/KafkaApis.scala
@@ -19,13 +19,13 @@ package kafka.server
import java.nio.ByteBuffer
import java.lang.{Long => JLong, Short => JShort}
+import java.util
import java.util.Properties
import kafka.admin.{AdminUtils, RackAwareMode}
-import kafka.api._
+import kafka.api.{ControlledShutdownRequest, ControlledShutdownResponse, FetchResponsePartitionData}
import kafka.cluster.Partition
-import kafka.server.QuotaFactory.{UnboundedQuota, QuotaManagers}
-import kafka.common
+import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota}
import kafka.common._
import kafka.controller.KafkaController
import kafka.coordinator.{GroupCoordinator, JoinGroupResult}
@@ -34,12 +34,13 @@ import kafka.message.{ByteBufferMessageSet, Message, MessageSet}
import kafka.network._
import kafka.network.RequestChannel.{Response, Session}
import kafka.security.auth
-import kafka.security.auth.{Authorizer, ClusterAction, Create, Describe, Group, Operation, Read, Resource, Write, Delete}
+import kafka.security.auth.{Authorizer, ClusterAction, Create, Delete, Describe, Group, Operation, Read, Resource, Write}
import kafka.utils.{Logging, SystemTime, ZKGroupTopicDirs, ZkUtils}
-import org.apache.kafka.common.errors.{ClusterAuthorizationException, NotLeaderForPartitionException, UnknownTopicOrPartitionException, TopicExistsException, UnsupportedForMessageFormatException}
+import org.apache.kafka.common.errors.{ClusterAuthorizationException, NotLeaderForPartitionException, TopicExistsException, UnknownTopicOrPartitionException, UnsupportedForMessageFormatException}
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.protocol.{ApiKeys, Errors, Protocol, SecurityProtocol}
-import org.apache.kafka.common.requests.{ApiVersionsResponse, DescribeGroupsRequest, DescribeGroupsResponse, GroupCoordinatorRequest, GroupCoordinatorResponse, HeartbeatRequest, HeartbeatResponse, JoinGroupRequest, JoinGroupResponse, LeaderAndIsrRequest, LeaderAndIsrResponse, LeaveGroupRequest, LeaveGroupResponse, ListGroupsResponse, ListOffsetRequest, ListOffsetResponse, MetadataRequest, MetadataResponse, OffsetCommitRequest, OffsetCommitResponse, OffsetFetchRequest, OffsetFetchResponse, ProduceRequest, ProduceResponse, ResponseHeader, ResponseSend, StopReplicaRequest, StopReplicaResponse, SyncGroupRequest, SyncGroupResponse, UpdateMetadataRequest, UpdateMetadataResponse, CreateTopicsRequest, CreateTopicsResponse, DeleteTopicsRequest, DeleteTopicsResponse}
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
import org.apache.kafka.common.utils.Utils
import org.apache.kafka.common.{Node, TopicPartition}
@@ -105,14 +106,13 @@ class KafkaApis(val requestChannel: RequestChannel,
error("Error when handling request %s".format(request.requestObj), e)
} else {
val response = request.body.getErrorResponse(request.header.apiVersion, e)
- val respHeader = new ResponseHeader(request.header.correlationId)
/* If request doesn't have a default error response, we just close the connection.
For example, when produce request has acks set to 0 */
if (response == null)
requestChannel.closeConnection(request.processor, request)
else
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, response)))
+ requestChannel.sendResponse(new Response(request, response))
error("Error when handling request %s".format(request.body), e)
}
@@ -142,7 +142,6 @@ class KafkaApis(val requestChannel: RequestChannel,
}
}
- val responseHeader = new ResponseHeader(correlationId)
val leaderAndIsrResponse =
if (authorize(request.session, ClusterAction, Resource.ClusterResource)) {
val result = replicaManager.becomeLeaderOrFollower(correlationId, leaderAndIsrRequest, metadataCache, onLeadershipChange)
@@ -152,7 +151,7 @@ class KafkaApis(val requestChannel: RequestChannel,
new LeaderAndIsrResponse(Errors.CLUSTER_AUTHORIZATION_FAILED.code, result.asJava)
}
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, responseHeader, leaderAndIsrResponse)))
+ requestChannel.sendResponse(new Response(request, leaderAndIsrResponse))
} catch {
case e: KafkaStorageException =>
fatal("Disk error during leadership change.", e)
@@ -166,7 +165,6 @@ class KafkaApis(val requestChannel: RequestChannel,
// stop serving data to clients for the topic being deleted
val stopReplicaRequest = request.body.asInstanceOf[StopReplicaRequest]
- val responseHeader = new ResponseHeader(request.header.correlationId)
val response =
if (authorize(request.session, ClusterAction, Resource.ClusterResource)) {
val (result, error) = replicaManager.stopReplicas(stopReplicaRequest)
@@ -176,7 +174,7 @@ class KafkaApis(val requestChannel: RequestChannel,
new StopReplicaResponse(Errors.CLUSTER_AUTHORIZATION_FAILED.code, result.asJava)
}
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, response)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads()
}
@@ -197,8 +195,7 @@ class KafkaApis(val requestChannel: RequestChannel,
new UpdateMetadataResponse(Errors.CLUSTER_AUTHORIZATION_FAILED.code)
}
- val responseHeader = new ResponseHeader(correlationId)
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, responseHeader, updateMetadataResponse)))
+ requestChannel.sendResponse(new Response(request, updateMetadataResponse))
}
def handleControlledShutdownRequest(request: RequestChannel.Request) {
@@ -228,9 +225,8 @@ class KafkaApis(val requestChannel: RequestChannel,
val results = offsetCommitRequest.offsetData.keySet.asScala.map { topicPartition =>
(topicPartition, errorCode)
}.toMap
- val responseHeader = new ResponseHeader(header.correlationId)
- val responseBody = new OffsetCommitResponse(results.asJava)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ val response = new OffsetCommitResponse(results.asJava)
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
} else {
val (existingAndAuthorizedForDescribeTopics, nonExistingOrUnauthorizedForDescribeTopics) = offsetCommitRequest.offsetData.asScala.toMap.partition {
case (topicPartition, _) => {
@@ -260,9 +256,8 @@ class KafkaApis(val requestChannel: RequestChannel,
s"on partition $topicPartition failed due to ${Errors.forCode(errorCode).exceptionName}")
}
}
- val responseHeader = new ResponseHeader(header.correlationId)
- val responseBody = new OffsetCommitResponse(combinedCommitStatus.asJava)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ val response = new OffsetCommitResponse(combinedCommitStatus.asJava)
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
}
if (authorizedTopics.isEmpty)
@@ -330,7 +325,7 @@ class KafkaApis(val requestChannel: RequestChannel,
}
private def authorize(session: Session, operation: Operation, resource: Resource): Boolean =
- authorizer.map(_.authorize(session, operation, resource)).getOrElse(true)
+ authorizer.forall(_.authorize(session, operation, resource))
/**
* Handle a produce request
@@ -388,7 +383,6 @@ class KafkaApis(val requestChannel: RequestChannel,
requestChannel.noOperation(request.processor, request)
}
} else {
- val respHeader = new ResponseHeader(request.header.correlationId)
val respBody = request.header.apiVersion match {
case 0 => new ProduceResponse(mergedResponseStatus.asJava)
case version@(1 | 2) => new ProduceResponse(mergedResponseStatus.asJava, delayTimeMs, version)
@@ -397,7 +391,7 @@ class KafkaApis(val requestChannel: RequestChannel,
case version => throw new IllegalArgumentException(s"Version `$version` of ProduceRequest is not handled. Code must be updated.")
}
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, respBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, respBody))
}
}
@@ -416,9 +410,9 @@ class KafkaApis(val requestChannel: RequestChannel,
else {
val internalTopicsAllowed = request.header.clientId == AdminUtils.AdminClientId
- // Convert ByteBuffer to ByteBufferMessageSet
+ // Convert Records to ByteBufferMessageSet
val authorizedMessagesPerPartition = authorizedRequestInfo.map {
- case (topicPartition, buffer) => (topicPartition, new ByteBufferMessageSet(buffer))
+ case (topicPartition, records) => (topicPartition, new ByteBufferMessageSet(records.buffer))
}
// call the replica manager to append messages to the replicas
@@ -440,79 +434,85 @@ class KafkaApis(val requestChannel: RequestChannel,
* Handle a fetch request
*/
def handleFetchRequest(request: RequestChannel.Request) {
- val fetchRequest = request.requestObj.asInstanceOf[FetchRequest]
+ val fetchRequest = request.body.asInstanceOf[FetchRequest]
+ val versionId = request.header.apiVersion
+ val clientId = request.header.clientId
- val (existingAndAuthorizedForDescribeTopics, nonExistingOrUnauthorizedForDescribeTopics) = fetchRequest.requestInfo.partition {
- case (topicAndPartition, _) => authorize(request.session, Describe, new Resource(auth.Topic, topicAndPartition.topic)) && metadataCache.contains(topicAndPartition.topic)
+ val (existingAndAuthorizedForDescribeTopics, nonExistingOrUnauthorizedForDescribeTopics) = fetchRequest.fetchData.asScala.toSeq.partition {
+ case (tp, _) => authorize(request.session, Describe, new Resource(auth.Topic, tp.topic)) && metadataCache.contains(tp.topic)
}
val (authorizedRequestInfo, unauthorizedForReadRequestInfo) = existingAndAuthorizedForDescribeTopics.partition {
- case (topicAndPartition, _) => authorize(request.session, Read, new Resource(auth.Topic, topicAndPartition.topic))
+ case (tp, _) => authorize(request.session, Read, new Resource(auth.Topic, tp.topic))
}
- val nonExistingOrUnauthorizedForDescribePartitionData = nonExistingOrUnauthorizedForDescribeTopics.map { case (tp, _) =>
- (tp, FetchResponsePartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION.code, -1, MessageSet.Empty))
+ val nonExistingOrUnauthorizedForDescribePartitionData = nonExistingOrUnauthorizedForDescribeTopics.map {
+ case (tp, _) => (tp, new FetchResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION.code, -1, MemoryRecords.EMPTY))
}
- val unauthorizedForReadPartitionData = unauthorizedForReadRequestInfo.map { case (tp, _) =>
- (tp, FetchResponsePartitionData(Errors.TOPIC_AUTHORIZATION_FAILED.code, -1, MessageSet.Empty))
+
+ val unauthorizedForReadPartitionData = unauthorizedForReadRequestInfo.map {
+ case (tp, _) => (tp, new FetchResponse.PartitionData(Errors.TOPIC_AUTHORIZATION_FAILED.code, -1, MemoryRecords.EMPTY))
}
// the callback for sending a fetch response
def sendResponseCallback(responsePartitionData: Seq[(TopicAndPartition, FetchResponsePartitionData)]) {
-
- val convertedPartitionData =
+ val convertedPartitionData = {
// Need to down-convert message when consumer only takes magic value 0.
- if (fetchRequest.versionId <= 1) {
- responsePartitionData.map { case (tp, data) =>
-
- // We only do down-conversion when:
- // 1. The message format version configured for the topic is using magic value > 0, and
- // 2. The message set contains message whose magic > 0
- // This is to reduce the message format conversion as much as possible. The conversion will only occur
- // when new message format is used for the topic and we see an old request.
- // Please note that if the message format is changed from a higher version back to lower version this
- // test might break because some messages in new message format can be delivered to consumers before 0.10.0.0
- // without format down conversion.
- val convertedData = if (replicaManager.getMessageFormatVersion(tp).exists(_ > Message.MagicValue_V0) &&
- !data.messages.isMagicValueInAllWrapperMessages(Message.MagicValue_V0)) {
- trace(s"Down converting message to V0 for fetch request from ${fetchRequest.clientId}")
- new FetchResponsePartitionData(data.error, data.hw, data.messages.asInstanceOf[FileMessageSet].toMessageFormat(Message.MagicValue_V0))
- } else data
-
- tp -> convertedData
- }
- } else responsePartitionData
+ responsePartitionData.map { case (tp, data) =>
+
+ // We only do down-conversion when:
+ // 1. The message format version configured for the topic is using magic value > 0, and
+ // 2. The message set contains message whose magic > 0
+ // This is to reduce the message format conversion as much as possible. The conversion will only occur
+ // when new message format is used for the topic and we see an old request.
+ // Please note that if the message format is changed from a higher version back to lower version this
+ // test might break because some messages in new message format can be delivered to consumers before 0.10.0.0
+ // without format down conversion.
+ val convertedData = if (versionId <= 1 && replicaManager.getMessageFormatVersion(tp).exists(_ > Message.MagicValue_V0) &&
+ !data.messages.isMagicValueInAllWrapperMessages(Message.MagicValue_V0)) {
+ trace(s"Down converting message to V0 for fetch request from $clientId")
+ new FetchResponsePartitionData(data.error, data.hw, data.messages.asInstanceOf[FileMessageSet].toMessageFormat(Message.MagicValue_V0))
+ } else data
+
+ val records = convertedData.messages.asRecords
+ new TopicPartition(tp.topic, tp.partition) -> new FetchResponse.PartitionData(convertedData.error, convertedData.hw, records)
+ }
+ }
val mergedPartitionData = convertedPartitionData ++ unauthorizedForReadPartitionData ++ nonExistingOrUnauthorizedForDescribePartitionData
+ val fetchedPartitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData]()
+
+ mergedPartitionData.foreach { case (topicPartition, data) =>
+ if (data.errorCode != Errors.NONE.code)
+ debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " +
+ s"on partition $topicPartition failed due to ${Errors.forCode(data.errorCode).exceptionName}")
+
+ fetchedPartitionData.put(topicPartition, data)
- mergedPartitionData.foreach { case (topicAndPartition, data) =>
- if (data.error != Errors.NONE.code)
- debug(s"Fetch request with correlation id ${fetchRequest.correlationId} from client ${fetchRequest.clientId} " +
- s"on partition $topicAndPartition failed due to ${Errors.forCode(data.error).exceptionName}")
// record the bytes out metrics only when the response is being sent
- BrokerTopicStats.getBrokerTopicStats(topicAndPartition.topic).bytesOutRate.mark(data.messages.sizeInBytes)
- BrokerTopicStats.getBrokerAllTopicsStats().bytesOutRate.mark(data.messages.sizeInBytes)
+ BrokerTopicStats.getBrokerTopicStats(topicPartition.topic).bytesOutRate.mark(data.records.sizeInBytes)
+ BrokerTopicStats.getBrokerAllTopicsStats().bytesOutRate.mark(data.records.sizeInBytes)
}
+ val response = new FetchResponse(versionId, fetchedPartitionData, 0)
+
def fetchResponseCallback(delayTimeMs: Int) {
- trace(s"Sending fetch response to client ${fetchRequest.clientId} of " +
- s"${convertedPartitionData.map { case (_, v) => v.messages.sizeInBytes }.sum} bytes")
- val response = FetchResponse(fetchRequest.correlationId, mergedPartitionData.toSeq, fetchRequest.versionId, delayTimeMs)
- requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.connectionId, response)))
+ trace(s"Sending fetch response to client $clientId of " +
+ s"${convertedPartitionData.map { case (_, v) => v.records.sizeInBytes }.sum} bytes")
+ val fetchResponse = if (delayTimeMs > 0) new FetchResponse(versionId, fetchedPartitionData, delayTimeMs) else response
+ requestChannel.sendResponse(new RequestChannel.Response(request, fetchResponse))
}
// When this callback is triggered, the remote API call has completed
request.apiRemoteCompleteTimeMs = SystemTime.milliseconds
if (fetchRequest.isFromFollower) {
- //We've already evaluated against the quota and are good to go. Just need to record it now.
- val responseSize = sizeOfThrottledPartitions(fetchRequest, mergedPartitionData, quotas.leader)
+ // We've already evaluated against the quota and are good to go. Just need to record it now.
+ val responseSize = sizeOfThrottledPartitions(versionId, fetchRequest, mergedPartitionData, quotas.leader)
quotas.leader.record(responseSize)
fetchResponseCallback(0)
} else {
- val responseSize = FetchResponse.responseSize(FetchResponse.batchByTopic(mergedPartitionData),
- fetchRequest.versionId)
- quotas.fetch.recordAndMaybeThrottle(request.session.sanitizedUser, fetchRequest.clientId, responseSize, fetchResponseCallback)
+ quotas.fetch.recordAndMaybeThrottle(request.session.sanitizedUser, clientId, response.sizeOf, fetchResponseCallback)
}
}
@@ -525,18 +525,23 @@ class KafkaApis(val requestChannel: RequestChannel,
fetchRequest.replicaId,
fetchRequest.minBytes,
fetchRequest.maxBytes,
- fetchRequest.versionId <= 2,
+ versionId <= 2,
authorizedRequestInfo,
replicationQuota(fetchRequest),
sendResponseCallback)
}
}
- private def sizeOfThrottledPartitions(fetchRequest: FetchRequest,
- mergedPartitionData: Seq[(TopicAndPartition, FetchResponsePartitionData)],
+ private def sizeOfThrottledPartitions(versionId: Short,
+ fetchRequest: FetchRequest,
+ mergedPartitionData: Seq[(TopicPartition, FetchResponse.PartitionData)],
quota: ReplicationQuotaManager): Int = {
- val throttledPartitions = mergedPartitionData.filter { case (partition, _) => quota.isThrottled(partition) }
- FetchResponse.responseSize(FetchRequest.batchByTopic(throttledPartitions), fetchRequest.versionId)
+ val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData]()
+ mergedPartitionData.foreach { case (tp, data) =>
+ if (quota.isThrottled(TopicAndPartition(tp.topic(), tp.partition())))
+ partitionData.put(tp, data)
+ }
+ FetchResponse.sizeOf(versionId, partitionData)
}
def replicationQuota(fetchRequest: FetchRequest): ReplicaQuota =
@@ -546,7 +551,6 @@ class KafkaApis(val requestChannel: RequestChannel,
* Handle an offset request
*/
def handleOffsetRequest(request: RequestChannel.Request) {
- val correlationId = request.header.correlationId
val version = request.header.apiVersion()
val mergedResponseMap =
@@ -555,10 +559,8 @@ class KafkaApis(val requestChannel: RequestChannel,
else
handleOffsetRequestV1(request)
- val responseHeader = new ResponseHeader(correlationId)
val response = new ListOffsetResponse(mergedResponseMap.asJava, version)
-
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, response)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
}
private def handleOffsetRequestV0(request : RequestChannel.Request) : Map[TopicPartition, ListOffsetResponse.PartitionData] = {
@@ -711,7 +713,7 @@ class KafkaApis(val requestChannel: RequestChannel,
else
offsetTimeArray = new Array[(Long, Long)](segsArray.length)
- for (i <- 0 until segsArray.length)
+ for (i <- segsArray.indices)
offsetTimeArray(i) = (segsArray(i).baseOffset, segsArray(i).lastModified)
if (lastSegmentHasSize)
offsetTimeArray(segsArray.length) = (log.logEndOffset, SystemTime.milliseconds)
@@ -837,7 +839,7 @@ class KafkaApis(val requestChannel: RequestChannel,
}
val unauthorizedForCreateTopicMetadata = unauthorizedForCreateTopics.map(topic =>
- new MetadataResponse.TopicMetadata(Errors.TOPIC_AUTHORIZATION_FAILED, topic, common.Topic.isInternal(topic),
+ new MetadataResponse.TopicMetadata(Errors.TOPIC_AUTHORIZATION_FAILED, topic, Topic.isInternal(topic),
java.util.Collections.emptyList()))
// do not disclose the existence of topics unauthorized for Describe, so we've not even checked if they exist or not
@@ -865,8 +867,6 @@ class KafkaApis(val requestChannel: RequestChannel,
trace("Sending topic metadata %s and brokers %s for correlation id %d to client %s".format(completeTopicMetadata.mkString(","),
brokers.mkString(","), request.header.correlationId, request.header.clientId))
- val responseHeader = new ResponseHeader(request.header.correlationId)
-
val responseBody = new MetadataResponse(
brokers.map(_.getNode(request.securityProtocol)).asJava,
clusterId,
@@ -874,7 +874,7 @@ class KafkaApis(val requestChannel: RequestChannel,
completeTopicMetadata.asJava,
requestVersion
)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
/*
@@ -884,7 +884,6 @@ class KafkaApis(val requestChannel: RequestChannel,
val header = request.header
val offsetFetchRequest = request.body.asInstanceOf[OffsetFetchRequest]
- val responseHeader = new ResponseHeader(header.correlationId)
val offsetFetchResponse =
// reject the request if not authorized to the group
if (!authorize(request.session, Read, new Resource(Group, offsetFetchRequest.groupId))) {
@@ -933,16 +932,15 @@ class KafkaApis(val requestChannel: RequestChannel,
}
trace(s"Sending offset fetch response $offsetFetchResponse for correlation id ${header.correlationId} to client ${header.clientId}.")
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, responseHeader, offsetFetchResponse)))
+ requestChannel.sendResponse(new Response(request, offsetFetchResponse))
}
def handleGroupCoordinatorRequest(request: RequestChannel.Request) {
val groupCoordinatorRequest = request.body.asInstanceOf[GroupCoordinatorRequest]
- val responseHeader = new ResponseHeader(request.header.correlationId)
if (!authorize(request.session, Describe, new Resource(Group, groupCoordinatorRequest.groupId))) {
val responseBody = new GroupCoordinatorResponse(Errors.GROUP_AUTHORIZATION_FAILED.code, Node.noNode)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
} else {
val partition = coordinator.partitionFor(groupCoordinatorRequest.groupId)
@@ -966,16 +964,14 @@ class KafkaApis(val requestChannel: RequestChannel,
trace("Sending consumer metadata %s for correlation id %d to client %s."
.format(responseBody, request.header.correlationId, request.header.clientId))
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
}
def handleDescribeGroupRequest(request: RequestChannel.Request) {
val describeRequest = request.body.asInstanceOf[DescribeGroupsRequest]
- val responseHeader = new ResponseHeader(request.header.correlationId)
- val groups = describeRequest.groupIds().asScala.map {
- case groupId =>
+ val groups = describeRequest.groupIds().asScala.map { groupId =>
if (!authorize(request.session, Describe, new Resource(Group, groupId))) {
groupId -> DescribeGroupsResponse.GroupMetadata.forError(Errors.GROUP_AUTHORIZATION_FAILED)
} else {
@@ -991,11 +987,10 @@ class KafkaApis(val requestChannel: RequestChannel,
}.toMap
val responseBody = new DescribeGroupsResponse(groups.asJava)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
def handleListGroupsRequest(request: RequestChannel.Request) {
- val responseHeader = new ResponseHeader(request.header.correlationId)
val responseBody = if (!authorize(request.session, Describe, Resource.ClusterResource)) {
ListGroupsResponse.fromError(Errors.CLUSTER_AUTHORIZATION_FAILED)
} else {
@@ -1003,14 +998,13 @@ class KafkaApis(val requestChannel: RequestChannel,
val allGroups = groups.map { group => new ListGroupsResponse.Group(group.groupId, group.protocolType) }
new ListGroupsResponse(error.code, allGroups.asJava)
}
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
def handleJoinGroupRequest(request: RequestChannel.Request) {
import JavaConversions._
val joinGroupRequest = request.body.asInstanceOf[JoinGroupRequest]
- val responseHeader = new ResponseHeader(request.header.correlationId)
// the callback for sending a join-group response
def sendResponseCallback(joinResult: JoinGroupResult) {
@@ -1020,7 +1014,7 @@ class KafkaApis(val requestChannel: RequestChannel,
trace("Sending join group response %s for correlation id %d to client %s."
.format(responseBody, request.header.correlationId, request.header.clientId))
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
if (!authorize(request.session, Read, new Resource(Group, joinGroupRequest.groupId()))) {
@@ -1032,7 +1026,7 @@ class KafkaApis(val requestChannel: RequestChannel,
JoinGroupResponse.UNKNOWN_MEMBER_ID, // memberId
JoinGroupResponse.UNKNOWN_MEMBER_ID, // leaderId
Map.empty[String, ByteBuffer])
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
} else {
// let the coordinator to handle join-group
val protocols = joinGroupRequest.groupProtocols().map(protocol =>
@@ -1057,8 +1051,7 @@ class KafkaApis(val requestChannel: RequestChannel,
def sendResponseCallback(memberState: Array[Byte], errorCode: Short) {
val responseBody = new SyncGroupResponse(errorCode, ByteBuffer.wrap(memberState))
- val responseHeader = new ResponseHeader(request.header.correlationId)
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new Response(request, responseBody))
}
if (!authorize(request.session, Read, new Resource(Group, syncGroupRequest.groupId()))) {
@@ -1068,7 +1061,7 @@ class KafkaApis(val requestChannel: RequestChannel,
syncGroupRequest.groupId(),
syncGroupRequest.generationId(),
syncGroupRequest.memberId(),
- syncGroupRequest.groupAssignment().mapValues(Utils.toArray(_)),
+ syncGroupRequest.groupAssignment().mapValues(Utils.toArray),
sendResponseCallback
)
}
@@ -1076,19 +1069,18 @@ class KafkaApis(val requestChannel: RequestChannel,
def handleHeartbeatRequest(request: RequestChannel.Request) {
val heartbeatRequest = request.body.asInstanceOf[HeartbeatRequest]
- val respHeader = new ResponseHeader(request.header.correlationId)
// the callback for sending a heartbeat response
def sendResponseCallback(errorCode: Short) {
val response = new HeartbeatResponse(errorCode)
trace("Sending heartbeat response %s for correlation id %d to client %s."
.format(response, request.header.correlationId, request.header.clientId))
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, response)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
}
if (!authorize(request.session, Read, new Resource(Group, heartbeatRequest.groupId))) {
val heartbeatResponse = new HeartbeatResponse(Errors.GROUP_AUTHORIZATION_FAILED.code)
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, heartbeatResponse)))
+ requestChannel.sendResponse(new Response(request, heartbeatResponse))
}
else {
// let the coordinator to handle heartbeat
@@ -1102,19 +1094,18 @@ class KafkaApis(val requestChannel: RequestChannel,
def handleLeaveGroupRequest(request: RequestChannel.Request) {
val leaveGroupRequest = request.body.asInstanceOf[LeaveGroupRequest]
- val respHeader = new ResponseHeader(request.header.correlationId)
// the callback for sending a leave-group response
def sendResponseCallback(errorCode: Short) {
val response = new LeaveGroupResponse(errorCode)
trace("Sending leave group response %s for correlation id %d to client %s."
.format(response, request.header.correlationId, request.header.clientId))
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, response)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
}
if (!authorize(request.session, Read, new Resource(Group, leaveGroupRequest.groupId))) {
val leaveGroupResponse = new LeaveGroupResponse(Errors.GROUP_AUTHORIZATION_FAILED.code)
- requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, leaveGroupResponse)))
+ requestChannel.sendResponse(new Response(request, leaveGroupResponse))
} else {
// let the coordinator to handle leave-group
coordinator.handleLeaveGroup(
@@ -1125,9 +1116,8 @@ class KafkaApis(val requestChannel: RequestChannel,
}
def handleSaslHandshakeRequest(request: RequestChannel.Request) {
- val respHeader = new ResponseHeader(request.header.correlationId)
val response = new SaslHandshakeResponse(Errors.ILLEGAL_SASL_STATE.code, config.saslEnabledMechanisms)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, response)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, response))
}
def handleApiVersionsRequest(request: RequestChannel.Request) {
@@ -1137,12 +1127,11 @@ class KafkaApis(val requestChannel: RequestChannel,
// If this is considered to leak information about the broker version a workaround is to use SSL
// with client authentication which is performed at an earlier stage of the connection where the
// ApiVersionRequest is not available.
- val responseHeader = new ResponseHeader(request.header.correlationId)
val responseBody = if (Protocol.apiVersionSupported(ApiKeys.API_VERSIONS.id, request.header.apiVersion))
ApiVersionsResponse.apiVersionsResponse
else
ApiVersionsResponse.fromError(Errors.UNSUPPORTED_VERSION)
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, responseHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
def close() {
@@ -1154,10 +1143,9 @@ class KafkaApis(val requestChannel: RequestChannel,
val createTopicsRequest = request.body.asInstanceOf[CreateTopicsRequest]
def sendResponseCallback(results: Map[String, Errors]): Unit = {
- val respHeader = new ResponseHeader(request.header.correlationId)
val responseBody = new CreateTopicsResponse(results.asJava)
trace(s"Sending create topics response $responseBody for correlation id ${request.header.correlationId} to client ${request.header.clientId}.")
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
if (!controller.isActive()) {
@@ -1205,14 +1193,13 @@ class KafkaApis(val requestChannel: RequestChannel,
def sendResponseCallback(results: Map[String, Errors]): Unit = {
val completeResults = nonExistingOrUnauthorizedForDescribeTopics.map(topic => (topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)).toMap ++
unauthorizedForDeleteTopics.map(topic => (topic, Errors.TOPIC_AUTHORIZATION_FAILED)).toMap ++ results
- val respHeader = new ResponseHeader(request.header.correlationId)
val responseBody = new DeleteTopicsResponse(completeResults.asJava)
trace(s"Sending delete topics response $responseBody for correlation id ${request.header.correlationId} to client ${request.header.clientId}.")
- requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, responseBody)))
+ requestChannel.sendResponse(new RequestChannel.Response(request, responseBody))
}
if (!controller.isActive()) {
- val results = deleteTopicRequest.topics.asScala.map { case topic =>
+ val results = deleteTopicRequest.topics.asScala.map { topic =>
(topic, Errors.NOT_CONTROLLER)
}.toMap
sendResponseCallback(results)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/KafkaServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala
index 1c7f06a..4026a7e 100755
--- a/core/src/main/scala/kafka/server/KafkaServer.scala
+++ b/core/src/main/scala/kafka/server/KafkaServer.scala
@@ -17,45 +17,37 @@
package kafka.server
+import java.io.{File, IOException}
import java.net.SocketTimeoutException
import java.util
-
-import kafka.admin._
-import kafka.api.KAFKA_0_9_0
-import kafka.log.LogConfig
-import kafka.log.CleanerConfig
-import kafka.log.LogManager
import java.util.concurrent._
-import atomic.{AtomicBoolean, AtomicInteger}
-import java.io.{File, IOException}
-import java.nio.charset.StandardCharsets
-import java.util.UUID
-import javax.xml.bind.DatatypeConverter
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
+import com.yammer.metrics.core.Gauge
+import kafka.admin.AdminUtils
+import kafka.api.KAFKA_0_9_0
+import kafka.cluster.{Broker, EndPoint}
+import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException}
+import kafka.controller.{ControllerStats, KafkaController}
+import kafka.coordinator.GroupCoordinator
+import kafka.log.{CleanerConfig, LogConfig, LogManager}
+import kafka.metrics.{KafkaMetricsGroup, KafkaMetricsReporter}
+import kafka.network.{BlockingChannel, SocketServer}
import kafka.security.auth.Authorizer
import kafka.utils._
+import org.I0Itec.zkclient.ZkClient
import org.apache.kafka.clients.{ClientRequest, ManualMetadataUpdater, NetworkClient}
-import org.apache.kafka.common.{ClusterResource, Node}
-import org.apache.kafka.common.metrics._
-import org.apache.kafka.common.network.{ChannelBuilders, LoginType, Mode, NetworkReceive, Selectable, Selector}
+import org.apache.kafka.common.internals.ClusterResourceListeners
+import org.apache.kafka.common.metrics.{JmxReporter, Metrics, _}
+import org.apache.kafka.common.network._
import org.apache.kafka.common.protocol.{ApiKeys, Errors, SecurityProtocol}
-import org.apache.kafka.common.metrics.{JmxReporter, Metrics}
-import org.apache.kafka.common.requests.{ControlledShutdownRequest, ControlledShutdownResponse, RequestSend}
+import org.apache.kafka.common.requests.{ControlledShutdownRequest, ControlledShutdownResponse}
import org.apache.kafka.common.security.JaasUtils
import org.apache.kafka.common.utils.AppInfoParser
+import org.apache.kafka.common.{ClusterResource, Node}
-import scala.collection.{Map, mutable}
import scala.collection.JavaConverters._
-import org.I0Itec.zkclient.ZkClient
-import kafka.controller.{ControllerStats, KafkaController}
-import kafka.cluster.{Broker, EndPoint}
-import kafka.common.{GenerateBrokerIdException, InconsistentBrokerIdException}
-import kafka.network.{BlockingChannel, SocketServer}
-import kafka.metrics.{KafkaMetricsGroup, KafkaMetricsReporter}
-import com.yammer.metrics.core.Gauge
-import kafka.coordinator.GroupCoordinator
-import org.apache.kafka.common.internals.ClusterResourceListeners
-import collection.JavaConverters._
+import scala.collection.{Map, mutable}
object KafkaServer {
// Copy the subset of properties that are relevant to Logs
@@ -433,12 +425,12 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr
// send the controlled shutdown request
val requestHeader = networkClient.nextRequestHeader(ApiKeys.CONTROLLED_SHUTDOWN_KEY)
- val send = new RequestSend(node(prevController).idString, requestHeader,
- new ControlledShutdownRequest(config.brokerId).toStruct)
- val request = new ClientRequest(kafkaMetricsTime.milliseconds(), true, send, null)
- val clientResponse = networkClient.blockingSendAndReceive(request)
+ val controlledShutdownRequest = new ControlledShutdownRequest(config.brokerId)
+ val request = new ClientRequest(node(prevController).idString, kafkaMetricsTime.milliseconds(), true,
+ requestHeader, controlledShutdownRequest, null)
+ val clientResponse = networkClient.blockingSendAndReceive(request, controlledShutdownRequest)
- val shutdownResponse = new ControlledShutdownResponse(clientResponse.responseBody)
+ val shutdownResponse = clientResponse.responseBody.asInstanceOf[ControlledShutdownResponse]
if (shutdownResponse.errorCode == Errors.NONE.code && shutdownResponse.partitionsRemaining.isEmpty) {
shutdownSucceeded = true
info("Controlled shutdown succeeded")
@@ -707,7 +699,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr
for(logDir <- logDirsWithoutMetaProps) {
val checkpoint = brokerMetadataCheckpoints(logDir)
- checkpoint.write(new BrokerMetadata(brokerId))
+ checkpoint.write(BrokerMetadata(brokerId))
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
index 6f4c589..84c3feb 100644
--- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
+++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
@@ -29,15 +29,16 @@ import kafka.common.{KafkaStorageException, TopicAndPartition}
import ReplicaFetcherThread._
import org.apache.kafka.clients.{ClientRequest, ClientResponse, ManualMetadataUpdater, NetworkClient}
import org.apache.kafka.common.network.{ChannelBuilders, LoginType, Mode, NetworkReceive, Selectable, Selector}
-import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, ListOffsetRequest, ListOffsetResponse, RequestSend}
+import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, ListOffsetRequest, ListOffsetResponse}
import org.apache.kafka.common.requests.{FetchRequest => JFetchRequest}
import org.apache.kafka.common.{Node, TopicPartition}
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.protocol.{ApiKeys, Errors}
+import org.apache.kafka.common.record.MemoryRecords
import org.apache.kafka.common.utils.Time
-import scala.collection.{JavaConverters, Map}
-import JavaConverters._
+import scala.collection.Map
+import scala.collection.JavaConverters._
class ReplicaFetcherThread(name: String,
fetcherId: Int,
@@ -139,7 +140,7 @@ class ReplicaFetcherThread(name: String,
if (logger.isTraceEnabled)
trace("Follower %d set replica high watermark for partition [%s,%d] to %s"
.format(replica.brokerId, topic, partitionId, followerHighWatermark))
- if (quota.isThrottled(new TopicAndPartition(topic, partitionId)))
+ if (quota.isThrottled(TopicAndPartition(topic, partitionId)))
quota.record(messageSet.sizeInBytes)
} catch {
case e: KafkaStorageException =>
@@ -236,7 +237,8 @@ class ReplicaFetcherThread(name: String,
protected def fetch(fetchRequest: FetchRequest): Seq[(TopicPartition, PartitionData)] = {
val clientResponse = sendRequest(ApiKeys.FETCH, Some(fetchRequestVersion), fetchRequest.underlying)
- new FetchResponse(clientResponse.responseBody).responseData.asScala.toSeq.map { case (key, value) =>
+ val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse]
+ fetchResponse.responseData.asScala.toSeq.map { case (key, value) =>
key -> new PartitionData(value)
}
}
@@ -248,9 +250,8 @@ class ReplicaFetcherThread(name: String,
if (!networkClient.blockingReady(sourceNode, socketTimeout)(time))
throw new SocketTimeoutException(s"Failed to connect within $socketTimeout ms")
else {
- val send = new RequestSend(sourceBroker.id.toString, header, request.toStruct)
- val clientRequest = new ClientRequest(time.milliseconds(), true, send, null)
- networkClient.blockingSendAndReceive(clientRequest)(time)
+ val clientRequest = new ClientRequest(sourceBroker.id.toString, time.milliseconds(), true, header, request, null)
+ networkClient.blockingSendAndReceive(clientRequest, request)(time)
}
}
catch {
@@ -271,7 +272,7 @@ class ReplicaFetcherThread(name: String,
(new ListOffsetRequest(consumerId, partitions.asJava), 0)
}
val clientResponse = sendRequest(ApiKeys.LIST_OFFSETS, Some(apiVersion.toShort), request)
- val response = new ListOffsetResponse(clientResponse.responseBody)
+ val response = clientResponse.responseBody.asInstanceOf[ListOffsetResponse]
val partitionData = response.responseData.get(topicPartition)
Errors.forCode(partitionData.errorCode) match {
case Errors.NONE =>
@@ -322,7 +323,10 @@ object ReplicaFetcherThread {
def errorCode: Short = underlying.errorCode
- def toByteBufferMessageSet: ByteBufferMessageSet = new ByteBufferMessageSet(underlying.recordSet)
+ def toByteBufferMessageSet: ByteBufferMessageSet = {
+ val buffer = underlying.records.asInstanceOf[MemoryRecords].buffer
+ new ByteBufferMessageSet(buffer)
+ }
def highWatermark: Long = underlying.highWatermark
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/ReplicaManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala
index b43695a..febe8ad 100644
--- a/core/src/main/scala/kafka/server/ReplicaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicaManager.scala
@@ -36,6 +36,7 @@ import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.requests.{LeaderAndIsrRequest, PartitionState, StopReplicaRequest, UpdateMetadataRequest}
import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
+import org.apache.kafka.common.requests.FetchRequest.PartitionData
import org.apache.kafka.common.utils.{Time => JTime}
import scala.collection._
@@ -457,7 +458,7 @@ class ReplicaManager(val config: KafkaConfig,
fetchMinBytes: Int,
fetchMaxBytes: Int,
hardMaxBytesLimit: Boolean,
- fetchInfos: Seq[(TopicAndPartition, PartitionFetchInfo)],
+ fetchInfos: Seq[(TopicPartition, PartitionData)],
quota: ReplicaQuota = UnboundedQuota,
responseCallback: Seq[(TopicAndPartition, FetchResponsePartitionData)] => Unit) {
val isFromFollower = replicaId >= 0
@@ -498,7 +499,7 @@ class ReplicaManager(val config: KafkaConfig,
// construct the fetch results from the read results
val fetchPartitionStatus = logReadResults.map { case (topicAndPartition, result) =>
val fetchInfo = fetchInfos.collectFirst {
- case (tp, v) if tp == topicAndPartition => v
+ case (tp, v) if TopicAndPartition(tp.topic, tp.partition) == topicAndPartition => v
}.getOrElse(sys.error(s"Partition $topicAndPartition not found in fetchInfos"))
(topicAndPartition, FetchPartitionStatus(result.info.fetchOffsetMetadata, fetchInfo))
}
@@ -524,19 +525,21 @@ class ReplicaManager(val config: KafkaConfig,
readOnlyCommitted: Boolean,
fetchMaxBytes: Int,
hardMaxBytesLimit: Boolean,
- readPartitionInfo: Seq[(TopicAndPartition, PartitionFetchInfo)],
+ readPartitionInfo: Seq[(TopicPartition, PartitionData)],
quota: ReplicaQuota): Seq[(TopicAndPartition, LogReadResult)] = {
- def read(tp: TopicAndPartition, fetchInfo: PartitionFetchInfo, limitBytes: Int, minOneMessage: Boolean): LogReadResult = {
- val TopicAndPartition(topic, partition) = tp
- val PartitionFetchInfo(offset, fetchSize) = fetchInfo
+ def read(tp: TopicPartition, fetchInfo: PartitionData, limitBytes: Int, minOneMessage: Boolean): LogReadResult = {
+ val topic = tp.topic
+ val partition = tp.partition
+ val offset = fetchInfo.offset
+ val partitionFetchSize = fetchInfo.maxBytes
BrokerTopicStats.getBrokerTopicStats(topic).totalFetchRequestRate.mark()
BrokerTopicStats.getBrokerAllTopicsStats().totalFetchRequestRate.mark()
try {
- trace(s"Fetching log segment for partition $tp, offset ${offset}, partition fetch size ${fetchSize}, " +
- s"remaining response limit ${limitBytes}" +
+ trace(s"Fetching log segment for partition $tp, offset $offset, partition fetch size $partitionFetchSize, " +
+ s"remaining response limit $limitBytes" +
(if (minOneMessage) s", ignoring response/partition size limits" else ""))
// decide whether to only fetch from leader
@@ -560,13 +563,13 @@ class ReplicaManager(val config: KafkaConfig,
val initialLogEndOffset = localReplica.logEndOffset
val logReadInfo = localReplica.log match {
case Some(log) =>
- val adjustedFetchSize = math.min(fetchSize, limitBytes)
+ val adjustedFetchSize = math.min(partitionFetchSize, limitBytes)
// Try the read first, this tells us whether we need all of adjustedFetchSize for this partition
val fetch = log.read(offset, adjustedFetchSize, maxOffsetOpt, minOneMessage)
// If the partition is being throttled, simply return an empty set.
- if (shouldLeaderThrottle(quota, tp, replicaId))
+ if (shouldLeaderThrottle(quota, TopicAndPartition(tp.topic, tp.partition), replicaId))
FetchDataInfo(fetch.fetchOffsetMetadata, MessageSet.Empty)
// For FetchRequest version 3, we replace incomplete message sets with an empty one as consumers can make
// progress in such cases and don't need to report a `RecordTooLargeException`
@@ -581,7 +584,7 @@ class ReplicaManager(val config: KafkaConfig,
val readToEndOfLog = initialLogEndOffset.messageOffset - logReadInfo.fetchOffsetMetadata.messageOffset <= 0
- LogReadResult(logReadInfo, localReplica.highWatermark.messageOffset, fetchSize, readToEndOfLog, None)
+ LogReadResult(logReadInfo, localReplica.highWatermark.messageOffset, partitionFetchSize, readToEndOfLog, None)
} catch {
// NOTE: Failed fetch requests metric is not incremented for known exceptions since it
// is supposed to indicate un-expected failure of a broker in handling a fetch request
@@ -589,12 +592,12 @@ class ReplicaManager(val config: KafkaConfig,
_: NotLeaderForPartitionException |
_: ReplicaNotAvailableException |
_: OffsetOutOfRangeException) =>
- LogReadResult(FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MessageSet.Empty), -1L, fetchSize, false, Some(e))
+ LogReadResult(FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MessageSet.Empty), -1L, partitionFetchSize, false, Some(e))
case e: Throwable =>
BrokerTopicStats.getBrokerTopicStats(topic).failedFetchRequestRate.mark()
BrokerTopicStats.getBrokerAllTopicsStats().failedFetchRequestRate.mark()
error(s"Error processing fetch operation on partition ${tp}, offset $offset", e)
- LogReadResult(FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MessageSet.Empty), -1L, fetchSize, false, Some(e))
+ LogReadResult(FetchDataInfo(LogOffsetMetadata.UnknownOffsetMetadata, MessageSet.Empty), -1L, partitionFetchSize, false, Some(e))
}
}
@@ -608,7 +611,7 @@ class ReplicaManager(val config: KafkaConfig,
if (messageSetSize > 0)
minOneMessage = false
limitBytes = math.max(0, limitBytes - messageSetSize)
- result += (tp -> readResult)
+ result += (TopicAndPartition(tp.topic, tp.partition) -> readResult)
}
result
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala b/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
index ba2fb92..d4c82d8 100644
--- a/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
+++ b/core/src/main/scala/kafka/server/ReplicationQuotaManager.scala
@@ -116,7 +116,7 @@ class ReplicationQuotaManager(val config: ReplicationQuotaManagerConfig,
override def isThrottled(topicPartition: TopicAndPartition): Boolean = {
val partitions = throttledPartitions.get(topicPartition.topic)
if (partitions != null)
- partitions.contains(topicPartition.partition) || (partitions eq AllReplicas)
+ (partitions eq AllReplicas) || partitions.contains(topicPartition.partition)
else false
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/utils/NetworkClientBlockingOps.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/utils/NetworkClientBlockingOps.scala b/core/src/main/scala/kafka/utils/NetworkClientBlockingOps.scala
index 9b0828f..220b6e1 100644
--- a/core/src/main/scala/kafka/utils/NetworkClientBlockingOps.scala
+++ b/core/src/main/scala/kafka/utils/NetworkClientBlockingOps.scala
@@ -18,12 +18,13 @@
package kafka.utils
import java.io.IOException
+
import org.apache.kafka.clients.{ClientRequest, ClientResponse, NetworkClient}
import org.apache.kafka.common.Node
+import org.apache.kafka.common.requests.AbstractRequest
import scala.annotation.tailrec
import scala.collection.JavaConverters._
-
import org.apache.kafka.common.utils.{Time => JTime}
object NetworkClientBlockingOps {
@@ -102,22 +103,19 @@ class NetworkClientBlockingOps(val client: NetworkClient) extends AnyVal {
* This method is useful for implementing blocking behaviour on top of the non-blocking `NetworkClient`, use it with
* care.
*/
- def blockingSendAndReceive(request: ClientRequest)(implicit time: JTime): ClientResponse = {
+ def blockingSendAndReceive(request: ClientRequest, body: AbstractRequest)(implicit time: JTime): ClientResponse = {
client.send(request, time.milliseconds())
pollContinuously { responses =>
val response = responses.find { response =>
- response.request.request.header.correlationId == request.request.header.correlationId
+ response.requestHeader.correlationId == request.header.correlationId
}
response.foreach { r =>
- if (r.wasDisconnected) {
- val destination = request.request.destination
- throw new IOException(s"Connection to $destination was disconnected before the response was read")
- }
+ if (r.wasDisconnected)
+ throw new IOException(s"Connection to ${request.destination} was disconnected before the response was read")
}
response
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
index 8502ae0..6eebbbf 100644
--- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala
@@ -38,9 +38,9 @@ import org.junit.{After, Assert, Before, Test}
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.Buffer
-
import org.apache.kafka.common.KafkaException
import kafka.admin.AdminUtils
+import org.apache.kafka.common.record.MemoryRecords
class AuthorizerIntegrationTest extends BaseRequestTest {
@@ -55,7 +55,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
val correlationId = 0
val clientId = "client-Id"
val tp = new TopicPartition(topic, part)
- val topicAndPartition = new TopicAndPartition(topic, part)
+ val topicAndPartition = TopicAndPartition(topic, part)
val group = "my-group"
val topicResource = new Resource(Topic, topic)
val groupResource = new Resource(Group, group)
@@ -181,7 +181,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
}
private def createProduceRequest = {
- new requests.ProduceRequest(1, 5000, collection.mutable.Map(tp -> ByteBuffer.wrap("test".getBytes)).asJava)
+ new requests.ProduceRequest(1, 5000, collection.mutable.Map(tp -> MemoryRecords.readableRecords(ByteBuffer.wrap("test".getBytes))).asJava)
}
private def createFetchRequest = {
@@ -766,10 +766,10 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
resources: Set[ResourceType],
isAuthorized: Boolean,
isAuthorizedTopicDescribe: Boolean,
- topicExists: Boolean = true): AbstractRequestResponse = {
+ topicExists: Boolean = true): AbstractResponse = {
val resp = send(request, apiKey)
- val response = RequestKeyToResponseDeserializer(apiKey).getMethod("parse", classOf[ByteBuffer]).invoke(null, resp).asInstanceOf[AbstractRequestResponse]
- val error = Errors.forCode(RequestKeyToErrorCode(apiKey).asInstanceOf[(AbstractRequestResponse) => Short](response))
+ val response = RequestKeyToResponseDeserializer(apiKey).getMethod("parse", classOf[ByteBuffer]).invoke(null, resp).asInstanceOf[AbstractResponse]
+ val error = Errors.forCode(RequestKeyToErrorCode(apiKey).asInstanceOf[(AbstractResponse) => Short](response))
val authorizationErrorCodes = resources.flatMap { resourceType =>
if (resourceType == Topic) {
@@ -786,13 +786,13 @@ class AuthorizerIntegrationTest extends BaseRequestTest {
if (topicExists)
if (isAuthorized)
- assertFalse(s"${apiKey} should be allowed. Found unexpected authorization error $error", authorizationErrorCodes.contains(error))
+ assertFalse(s"$apiKey should be allowed. Found unexpected authorization error $error", authorizationErrorCodes.contains(error))
else
- assertTrue(s"${apiKey} should be forbidden. Found error $error but expected one of $authorizationErrorCodes", authorizationErrorCodes.contains(error))
+ assertTrue(s"$apiKey should be forbidden. Found error $error but expected one of $authorizationErrorCodes", authorizationErrorCodes.contains(error))
else if (resources == Set(Topic))
- assertEquals(s"${apiKey} had an unexpected error", Errors.UNKNOWN_TOPIC_OR_PARTITION, error)
+ assertEquals(s"$apiKey had an unexpected error", Errors.UNKNOWN_TOPIC_OR_PARTITION, error)
else
- assertNotEquals(s"${apiKey} had an unexpected error", Errors.TOPIC_AUTHORIZATION_FAILED, error)
+ assertNotEquals(s"$apiKey had an unexpected error", Errors.TOPIC_AUTHORIZATION_FAILED, error)
response
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/message/BaseMessageSetTestCases.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/message/BaseMessageSetTestCases.scala b/core/src/test/scala/unit/kafka/message/BaseMessageSetTestCases.scala
index 0d86128..476a577 100644
--- a/core/src/test/scala/unit/kafka/message/BaseMessageSetTestCases.scala
+++ b/core/src/test/scala/unit/kafka/message/BaseMessageSetTestCases.scala
@@ -103,7 +103,7 @@ trait BaseMessageSetTestCases extends JUnitSuite {
var remaining = messageSetSize
var iterations = 0
while (remaining > 0) {
- remaining -= messageSet.writeTo(channel, messageSetSize - remaining, remaining)
+ remaining -= messageSet.asRecords.writeTo(channel, messageSetSize - remaining, remaining).toInt
iterations += 1
}
@@ -112,7 +112,7 @@ trait BaseMessageSetTestCases extends JUnitSuite {
}
def checkWriteToWithMessageSet(messageSet: MessageSet) {
- checkWriteWithMessageSet(messageSet, messageSet.writeTo(_, 0, messageSet.sizeInBytes))
+ checkWriteWithMessageSet(messageSet, messageSet.asRecords.writeTo(_, 0, messageSet.sizeInBytes))
}
def checkWriteWithMessageSet(set: MessageSet, write: GatheringByteChannel => Long) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
index da6f260..f7b5da5 100644
--- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
+++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala
@@ -31,10 +31,9 @@ import org.apache.kafka.common.security.auth.KafkaPrincipal
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.requests.{ProduceRequest, RequestHeader}
import org.apache.kafka.common.utils.SystemTime
-
import kafka.server.KafkaConfig
import kafka.utils.TestUtils
-
+import org.apache.kafka.common.record.MemoryRecords
import org.junit.Assert._
import org.junit._
import org.scalatest.junit.JUnitSuite
@@ -117,7 +116,7 @@ class SocketServerTest extends JUnitSuite {
val ack = 0: Short
val emptyHeader = new RequestHeader(apiKey, clientId, correlationId)
- val emptyRequest = new ProduceRequest(ack, ackTimeoutMs, new HashMap[TopicPartition, ByteBuffer]())
+ val emptyRequest = new ProduceRequest(ack, ackTimeoutMs, new HashMap[TopicPartition, MemoryRecords]())
val byteBuffer = ByteBuffer.allocate(emptyHeader.sizeOf + emptyRequest.sizeOf)
emptyHeader.writeTo(byteBuffer)
@@ -270,7 +269,7 @@ class SocketServerTest extends JUnitSuite {
val ackTimeoutMs = 10000
val ack = 0: Short
val emptyHeader = new RequestHeader(apiKey, clientId, correlationId)
- val emptyRequest = new ProduceRequest(ack, ackTimeoutMs, new HashMap[TopicPartition, ByteBuffer]())
+ val emptyRequest = new ProduceRequest(ack, ackTimeoutMs, new HashMap[TopicPartition, MemoryRecords]())
val byteBuffer = ByteBuffer.allocate(emptyHeader.sizeOf() + emptyRequest.sizeOf())
emptyHeader.writeTo(byteBuffer)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
index 52809b3..4205ccb 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestTest.scala
@@ -172,7 +172,7 @@ class CreateTopicsRequestTest extends BaseRequestTest {
private def duplicateFirstTopic(request: CreateTopicsRequest) = {
val struct = request.toStruct
val topics = struct.getArray("create_topic_requests")
- val firstTopic= topics(0).asInstanceOf[Struct]
+ val firstTopic = topics(0).asInstanceOf[Struct]
val newTopics = firstTopic :: topics.toList
struct.set("create_topic_requests", newTopics.toArray)
new CreateTopicsRequest(struct)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
index 2ccb7b8..a886609 100755
--- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala
@@ -22,13 +22,13 @@ import java.net.Socket
import java.nio.ByteBuffer
import kafka.integration.KafkaServerTestHarness
-
import kafka.network.SocketServer
import kafka.utils._
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.protocol.types.Type
import org.apache.kafka.common.protocol.{ApiKeys, SecurityProtocol}
-import org.apache.kafka.common.requests.{ProduceResponse, ResponseHeader, ProduceRequest}
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests.{ProduceRequest, ProduceResponse, ResponseHeader}
import org.junit.Assert._
import org.junit.Test
@@ -116,7 +116,8 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness {
val serializedBytes = {
val headerBytes = requestHeaderBytes(ApiKeys.PRODUCE.id, 2, null, correlationId)
val messageBytes = "message".getBytes
- val request = new ProduceRequest(1, 10000, Map(topicPartition -> ByteBuffer.wrap(messageBytes)).asJava)
+ val records = MemoryRecords.readableRecords(ByteBuffer.wrap(messageBytes))
+ val request = new ProduceRequest(1, 10000, Map(topicPartition -> records).asJava)
val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.sizeOf)
byteBuffer.put(headerBytes)
request.writeTo(byteBuffer)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index bb0e060..3d1b485 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -129,7 +129,7 @@ class FetchRequestTest extends BaseRequestTest {
val size3 = logEntries(partitionData3).map(_.size).sum
assertTrue(s"Expected $size3 to be smaller than $maxResponseBytes", size3 <= maxResponseBytes)
assertTrue(s"Expected $size3 to be larger than $maxPartitionBytes", size3 > maxPartitionBytes)
- assertTrue(maxPartitionBytes < MemoryRecords.readableRecords(partitionData3.recordSet).sizeInBytes)
+ assertTrue(maxPartitionBytes < partitionData3.records.sizeInBytes)
// 4. Partition with message larger than the response limit at the start of the list
val shuffledTopicPartitions4 = Seq(partitionWithLargeMessage2, partitionWithLargeMessage1) ++
@@ -146,7 +146,7 @@ class FetchRequestTest extends BaseRequestTest {
assertTrue(partitionData4.highWatermark > 0)
val size4 = logEntries(partitionData4).map(_.size).sum
assertTrue(s"Expected $size4 to be larger than $maxResponseBytes", size4 > maxResponseBytes)
- assertTrue(maxResponseBytes < MemoryRecords.readableRecords(partitionData4.recordSet).sizeInBytes)
+ assertTrue(maxResponseBytes < partitionData4.records.sizeInBytes)
}
@Test
@@ -160,12 +160,12 @@ class FetchRequestTest extends BaseRequestTest {
val partitionData = fetchResponse.responseData.get(topicPartition)
assertEquals(Errors.NONE.code, partitionData.errorCode)
assertTrue(partitionData.highWatermark > 0)
- assertEquals(maxPartitionBytes, MemoryRecords.readableRecords(partitionData.recordSet).sizeInBytes)
+ assertEquals(maxPartitionBytes, partitionData.records.sizeInBytes)
assertEquals(0, logEntries(partitionData).map(_.size).sum)
}
private def logEntries(partitionData: FetchResponse.PartitionData): Seq[LogEntry] = {
- val memoryRecords = MemoryRecords.readableRecords(partitionData.recordSet)
+ val memoryRecords = partitionData.records
memoryRecords.iterator.asScala.toIndexedSeq
}
@@ -181,7 +181,7 @@ class FetchRequestTest extends BaseRequestTest {
assertEquals(Errors.NONE.code, partitionData.errorCode)
assertTrue(partitionData.highWatermark > 0)
- val memoryRecords = MemoryRecords.readableRecords(partitionData.recordSet)
+ val memoryRecords = partitionData.records
responseBufferSize += memoryRecords.sizeInBytes
val messages = memoryRecords.iterator.asScala.toIndexedSeq
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
index 2e24aae..5726152 100755
--- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
+++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala
@@ -21,7 +21,7 @@ import org.apache.kafka.common.TopicPartition
import scala.collection.JavaConverters._
import kafka.api.LeaderAndIsr
-import org.apache.kafka.common.requests.{AbstractRequestResponse, LeaderAndIsrRequest, LeaderAndIsrResponse, PartitionState}
+import org.apache.kafka.common.requests._
import org.junit.Assert._
import kafka.utils.{CoreUtils, TestUtils}
import kafka.cluster.Broker
@@ -156,7 +156,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness {
}
}
- private def staleControllerEpochCallback(response: AbstractRequestResponse): Unit = {
+ private def staleControllerEpochCallback(response: AbstractResponse): Unit = {
val leaderAndIsrResponse = response.asInstanceOf[LeaderAndIsrResponse]
staleControllerEpochDetected = Errors.forCode(leaderAndIsrResponse.errorCode) match {
case Errors.STALE_CONTROLLER_EPOCH => true
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
index 9e4c800..bd74dee 100644
--- a/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ProduceRequestTest.scala
@@ -23,7 +23,7 @@ import kafka.utils.TestUtils
import org.apache.kafka.test.{TestUtils => JTestUtils}
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.protocol.{ApiKeys, Errors, ProtoUtils}
-import org.apache.kafka.common.record.{CompressionType, Record}
+import org.apache.kafka.common.record.{CompressionType, MemoryRecords, Record}
import org.apache.kafka.common.requests.{ProduceRequest, ProduceResponse}
import org.junit.Assert._
import org.junit.Test
@@ -42,7 +42,7 @@ class ProduceRequestTest extends BaseRequestTest {
def sendAndCheck(recordBuffer: ByteBuffer, expectedOffset: Long): ProduceResponse.PartitionResponse = {
val topicPartition = new TopicPartition("topic", partition)
- val partitionRecords = Map(topicPartition -> recordBuffer)
+ val partitionRecords = Map(topicPartition -> MemoryRecords.readableRecords(recordBuffer))
val produceResponse = sendProduceRequest(leader, new ProduceRequest(-1, 3000, partitionRecords.asJava))
assertEquals(1, produceResponse.responses.size)
val (tp, partitionResponse) = produceResponse.responses.asScala.head
@@ -78,7 +78,7 @@ class ProduceRequestTest extends BaseRequestTest {
// Change the lz4 checksum value so that it doesn't match the contents
recordBuffer.array.update(40, 0)
val topicPartition = new TopicPartition("topic", partition)
- val partitionRecords = Map(topicPartition -> recordBuffer)
+ val partitionRecords = Map(topicPartition -> MemoryRecords.readableRecords(recordBuffer))
val produceResponse = sendProduceRequest(leader, new ProduceRequest(-1, 3000, partitionRecords.asJava))
assertEquals(1, produceResponse.responses.size)
val (tp, partitionResponse) = produceResponse.responses.asScala.head
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
index 17e0516..e226833 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerQuotasTest.scala
@@ -20,20 +20,20 @@ package kafka.server
import java.util.Properties
import java.util.concurrent.atomic.AtomicBoolean
-import kafka.api._
import kafka.cluster.Replica
import kafka.common.TopicAndPartition
import kafka.log.Log
import kafka.message.{ByteBufferMessageSet, Message}
import kafka.utils._
+import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.utils.{MockTime => JMockTime}
+import org.apache.kafka.common.requests.FetchRequest.PartitionData
import org.easymock.EasyMock
import org.easymock.EasyMock._
import org.junit.Assert._
import org.junit.{After, Test}
-
class ReplicaManagerQuotasTest {
val configs = TestUtils.createBrokerConfigs(2, TestUtils.MockZkConnect).map(KafkaConfig.fromProps(_, new Properties()))
val time = new MockTime
@@ -42,7 +42,8 @@ class ReplicaManagerQuotasTest {
val message = new Message("some-data-in-a-message".getBytes())
val topicAndPartition1 = TopicAndPartition("test-topic", 1)
val topicAndPartition2 = TopicAndPartition("test-topic", 2)
- val fetchInfo = Seq(topicAndPartition1 -> PartitionFetchInfo(0, 100), topicAndPartition2 -> PartitionFetchInfo(0, 100))
+ val fetchInfo = Seq(new TopicPartition(topicAndPartition1.topic, topicAndPartition1.partition) -> new PartitionData(0, 100),
+ new TopicPartition(topicAndPartition2.topic, topicAndPartition2.partition) -> new PartitionData(0, 100))
var replicaManager: ReplicaManager = null
@Test
@@ -143,7 +144,7 @@ class ReplicaManagerQuotasTest {
fetch.find(_._1 == topicAndPartition2).get._2.info.messageSet.size)
}
- def setUpMocks(fetchInfo: Seq[(TopicAndPartition, PartitionFetchInfo)], message: Message = this.message, bothReplicasInSync: Boolean = false) {
+ def setUpMocks(fetchInfo: Seq[(TopicPartition, PartitionData)], message: Message = this.message, bothReplicasInSync: Boolean = false) {
val zkUtils = createNiceMock(classOf[ZkUtils])
val scheduler = createNiceMock(classOf[KafkaScheduler])
@@ -185,7 +186,7 @@ class ReplicaManagerQuotasTest {
partition.leaderReplicaIdOpt = Some(leaderReplica.brokerId)
val followerReplica = new Replica(configs.last.brokerId, partition, time, 0, Some(log))
val allReplicas = Set(leaderReplica, followerReplica)
- allReplicas.foreach(partition.addReplicaIfNotExists(_))
+ allReplicas.foreach(partition.addReplicaIfNotExists)
if (bothReplicasInSync) {
partition.inSyncReplicas = allReplicas
followerReplica.highWatermark = new LogOffsetMetadata(5)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index ca8d712..243e06e 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -21,7 +21,7 @@ package kafka.server
import java.io.File
import java.util.concurrent.atomic.AtomicBoolean
-import kafka.api.{FetchResponsePartitionData, PartitionFetchInfo}
+import kafka.api.FetchResponsePartitionData
import kafka.cluster.Broker
import kafka.common.TopicAndPartition
import kafka.message.{ByteBufferMessageSet, Message, MessageSet}
@@ -31,6 +31,7 @@ import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.protocol.Errors
import org.apache.kafka.common.requests.{LeaderAndIsrRequest, PartitionState}
import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
+import org.apache.kafka.common.requests.FetchRequest.PartitionData
import org.apache.kafka.common.utils.{MockTime => JMockTime}
import org.apache.kafka.common.{Node, TopicPartition}
import org.easymock.EasyMock
@@ -173,7 +174,7 @@ class ReplicaManagerTest {
fetchMinBytes = 100000,
fetchMaxBytes = Int.MaxValue,
hardMaxBytesLimit = false,
- fetchInfos = Seq(new TopicAndPartition(topic, 0) -> new PartitionFetchInfo(0, 100000)),
+ fetchInfos = Seq(new TopicPartition(topic, 0) -> new PartitionData(0, 100000)),
responseCallback = fetchCallback)
// Make this replica the follower
@@ -244,7 +245,7 @@ class ReplicaManagerTest {
fetchMinBytes = 0,
fetchMaxBytes = Int.MaxValue,
hardMaxBytesLimit = false,
- fetchInfos = Seq(new TopicAndPartition(topic, 0) -> new PartitionFetchInfo(1, 100000)),
+ fetchInfos = Seq(new TopicPartition(topic, 0) -> new PartitionData(1, 100000)),
responseCallback = fetchCallback)
@@ -260,7 +261,7 @@ class ReplicaManagerTest {
fetchMinBytes = 0,
fetchMaxBytes = Int.MaxValue,
hardMaxBytesLimit = false,
- fetchInfos = Seq(new TopicAndPartition(topic, 0) -> new PartitionFetchInfo(1, 100000)),
+ fetchInfos = Seq(new TopicPartition(topic, 0) -> new PartitionData(1, 100000)),
responseCallback = fetchCallback)
assertTrue(fetchCallbackFired)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
index cbd751b..340b05f 100644
--- a/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
+++ b/core/src/test/scala/unit/kafka/server/SimpleFetchTest.scala
@@ -19,19 +19,18 @@ package kafka.server
import kafka.api._
import kafka.utils._
import kafka.cluster.Replica
-import kafka.common.TopicAndPartition
import kafka.log.Log
import kafka.message.{ByteBufferMessageSet, Message, MessageSet}
import kafka.server.QuotaFactory.UnboundedQuota
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.utils.{MockTime => JMockTime}
-
-import org.junit.{Test, After, Before}
-
-import java.util.{Properties}
+import org.apache.kafka.common.requests.FetchRequest.PartitionData
+import org.junit.{After, Before, Test}
+import java.util.Properties
import java.util.concurrent.atomic.AtomicBoolean
-import collection.JavaConversions._
+import kafka.common.TopicAndPartition
+import org.apache.kafka.common.TopicPartition
import org.easymock.EasyMock
import org.junit.Assert._
@@ -63,7 +62,7 @@ class SimpleFetchTest {
val partitionId = 0
val topicAndPartition = TopicAndPartition(topic, partitionId)
- val fetchInfo = Seq(topicAndPartition -> PartitionFetchInfo(0, fetchSize))
+ val fetchInfo = Seq(new TopicPartition(topicAndPartition.topic, topicAndPartition.partition) -> new PartitionData(0, fetchSize))
var replicaManager: ReplicaManager = null
@@ -117,7 +116,7 @@ class SimpleFetchTest {
// add both of them to ISR
val allReplicas = List(leaderReplica, followerReplica)
- allReplicas.foreach(partition.addReplicaIfNotExists(_))
+ allReplicas.foreach(partition.addReplicaIfNotExists)
partition.inSyncReplicas = allReplicas.toSet
}
[4/5] kafka git commit: KAFKA-2066;
Use client-side FetchRequest/FetchResponse on server
Posted by jg...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/record/LogInputStream.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/LogInputStream.java b/clients/src/main/java/org/apache/kafka/common/record/LogInputStream.java
new file mode 100644
index 0000000..4a4d569
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/record/LogInputStream.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+package org.apache.kafka.common.record;
+
+import java.io.IOException;
+
+/**
+ * An abstraction between an underlying input stream and record iterators, a LogInputStream
+ * returns only the shallow log entries, depending on {@link org.apache.kafka.common.record.RecordsIterator.DeepRecordsIterator}
+ * for the deep iteration.
+ */
+interface LogInputStream {
+
+ /**
+ * Get the next log entry from the underlying input stream.
+ *
+ * @return The next log entry or null if there is none
+ * @throws IOException for any IO errors
+ */
+ LogEntry nextEntry() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
index 3848ea9..65ccf98 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
@@ -13,21 +13,18 @@
package org.apache.kafka.common.record;
import java.io.DataInputStream;
-import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayDeque;
+import java.nio.channels.GatheringByteChannel;
import java.util.Iterator;
-import org.apache.kafka.common.KafkaException;
-import org.apache.kafka.common.utils.AbstractIterator;
-import org.apache.kafka.common.utils.Utils;
-
/**
* A {@link Records} implementation backed by a ByteBuffer.
*/
public class MemoryRecords implements Records {
+ public final static MemoryRecords EMPTY = MemoryRecords.readableRecords(ByteBuffer.allocate(0));
+
private final static int WRITE_LIMIT_FOR_READABLE_ONLY = -1;
// the compressor used for appends-only
@@ -148,6 +145,7 @@ public class MemoryRecords implements Records {
/**
* The size of this record set
*/
+ @Override
public int sizeInBytes() {
if (writable) {
return compressor.buffer().position();
@@ -156,6 +154,15 @@ public class MemoryRecords implements Records {
}
}
+ @Override
+ public long writeTo(GatheringByteChannel channel, long offset, int length) throws IOException {
+ ByteBuffer dup = buffer.duplicate();
+ int position = (int) offset;
+ dup.position(position);
+ dup.limit(position + length);
+ return channel.write(dup);
+ }
+
/**
* The compression rate of this record set
*/
@@ -186,13 +193,11 @@ public class MemoryRecords implements Records {
@Override
public Iterator<LogEntry> iterator() {
- if (writable) {
+ ByteBuffer input = this.buffer.duplicate();
+ if (writable)
// flip on a duplicate buffer for reading
- return new RecordsIterator((ByteBuffer) this.buffer.duplicate().flip(), false);
- } else {
- // do not need to flip for non-writable buffer
- return new RecordsIterator(this.buffer.duplicate(), false);
- }
+ input.flip();
+ return new RecordsIterator(new ByteBufferLogInputStream(input), false);
}
@Override
@@ -219,151 +224,44 @@ public class MemoryRecords implements Records {
return writable;
}
- public static class RecordsIterator extends AbstractIterator<LogEntry> {
- private final ByteBuffer buffer;
- private final DataInputStream stream;
- private final CompressionType type;
- private final boolean shallow;
- private RecordsIterator innerIter;
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
- // The variables for inner iterator
- private final ArrayDeque<LogEntry> logEntries;
- private final long absoluteBaseOffset;
+ MemoryRecords that = (MemoryRecords) o;
- public RecordsIterator(ByteBuffer buffer, boolean shallow) {
- this.type = CompressionType.NONE;
- this.buffer = buffer;
- this.shallow = shallow;
- this.stream = new DataInputStream(new ByteBufferInputStream(buffer));
- this.logEntries = null;
- this.absoluteBaseOffset = -1;
- }
+ return buffer.equals(that.buffer);
- // Private constructor for inner iterator.
- private RecordsIterator(LogEntry entry) {
- this.type = entry.record().compressionType();
- this.buffer = entry.record().value();
- this.shallow = true;
- this.stream = Compressor.wrapForInput(new ByteBufferInputStream(this.buffer), type, entry.record().magic());
- long wrapperRecordOffset = entry.offset();
-
- long wrapperRecordTimestamp = entry.record().timestamp();
- this.logEntries = new ArrayDeque<>();
- // If relative offset is used, we need to decompress the entire message first to compute
- // the absolute offset. For simplicity and because it's a format that is on its way out, we
- // do the same for message format version 0
- try {
- while (true) {
- try {
- LogEntry logEntry = getNextEntryFromStream();
- if (entry.record().magic() > Record.MAGIC_VALUE_V0) {
- Record recordWithTimestamp = new Record(
- logEntry.record().buffer(),
- wrapperRecordTimestamp,
- entry.record().timestampType()
- );
- logEntry = new LogEntry(logEntry.offset(), recordWithTimestamp);
- }
- logEntries.add(logEntry);
- } catch (EOFException e) {
- break;
- }
- }
- if (entry.record().magic() > Record.MAGIC_VALUE_V0)
- this.absoluteBaseOffset = wrapperRecordOffset - logEntries.getLast().offset();
- else
- this.absoluteBaseOffset = -1;
- } catch (IOException e) {
- throw new KafkaException(e);
- } finally {
- Utils.closeQuietly(stream, "records iterator stream");
- }
- }
+ }
- /*
- * Read the next record from the buffer.
- *
- * Note that in the compressed message set, each message value size is set as the size of the un-compressed
- * version of the message value, so when we do de-compression allocating an array of the specified size for
- * reading compressed value data is sufficient.
- */
- @Override
- protected LogEntry makeNext() {
- if (innerDone()) {
- try {
- LogEntry entry = getNextEntry();
- // No more record to return.
- if (entry == null)
- return allDone();
-
- // Convert offset to absolute offset if needed.
- if (absoluteBaseOffset >= 0) {
- long absoluteOffset = absoluteBaseOffset + entry.offset();
- entry = new LogEntry(absoluteOffset, entry.record());
- }
-
- // decide whether to go shallow or deep iteration if it is compressed
- CompressionType compression = entry.record().compressionType();
- if (compression == CompressionType.NONE || shallow) {
- return entry;
- } else {
- // init the inner iterator with the value payload of the message,
- // which will de-compress the payload to a set of messages;
- // since we assume nested compression is not allowed, the deep iterator
- // would not try to further decompress underlying messages
- // There will be at least one element in the inner iterator, so we don't
- // need to call hasNext() here.
- innerIter = new RecordsIterator(entry);
- return innerIter.next();
- }
- } catch (EOFException e) {
- return allDone();
- } catch (IOException e) {
- throw new KafkaException(e);
- }
- } else {
- return innerIter.next();
- }
- }
+ @Override
+ public int hashCode() {
+ return buffer.hashCode();
+ }
- private LogEntry getNextEntry() throws IOException {
- if (logEntries != null)
- return getNextEntryFromEntryList();
- else
- return getNextEntryFromStream();
- }
+ private static class ByteBufferLogInputStream implements LogInputStream {
+ private final DataInputStream stream;
+ private final ByteBuffer buffer;
- private LogEntry getNextEntryFromEntryList() {
- return logEntries.isEmpty() ? null : logEntries.remove();
+ private ByteBufferLogInputStream(ByteBuffer buffer) {
+ this.stream = new DataInputStream(new ByteBufferInputStream(buffer));
+ this.buffer = buffer;
}
- private LogEntry getNextEntryFromStream() throws IOException {
- // read the offset
+ public LogEntry nextEntry() throws IOException {
long offset = stream.readLong();
- // read record size
int size = stream.readInt();
if (size < 0)
throw new IllegalStateException("Record with size " + size);
- // read the record, if compression is used we cannot depend on size
- // and hence has to do extra copy
- ByteBuffer rec;
- if (type == CompressionType.NONE) {
- rec = buffer.slice();
- int newPos = buffer.position() + size;
- if (newPos > buffer.limit())
- return null;
- buffer.position(newPos);
- rec.limit(size);
- } else {
- byte[] recordBuffer = new byte[size];
- stream.readFully(recordBuffer, 0, size);
- rec = ByteBuffer.wrap(recordBuffer);
- }
- return new LogEntry(offset, new Record(rec));
- }
- private boolean innerDone() {
- return innerIter == null || !innerIter.hasNext();
+ ByteBuffer slice = buffer.slice();
+ int newPos = buffer.position() + size;
+ if (newPos > buffer.limit())
+ return null;
+ buffer.position(newPos);
+ slice.limit(size);
+ return new LogEntry(offset, new Record(slice));
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/record/Records.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/Records.java b/clients/src/main/java/org/apache/kafka/common/record/Records.java
index d43cdab..3bc043f 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/Records.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/Records.java
@@ -16,6 +16,9 @@
*/
package org.apache.kafka.common.record;
+import java.io.IOException;
+import java.nio.channels.GatheringByteChannel;
+
/**
* A binary format which consists of a 4 byte size, an 8 byte offset, and the record bytes. See {@link MemoryRecords}
* for the in-memory representation.
@@ -28,7 +31,19 @@ public interface Records extends Iterable<LogEntry> {
/**
* The size of these records in bytes
+ * @return The size in bytes
*/
int sizeInBytes();
+ /**
+ * Write the messages in this set to the given channel starting at the given offset byte.
+ * @param channel The channel to write to
+ * @param position The position within this record set to begin writing from
+ * @param length The number of bytes to write
+ * @return The number of bytes written to the channel (which may be fewer than requested)
+ * @throws IOException For any IO errors copying the
+ */
+ long writeTo(GatheringByteChannel channel, long position, int length) throws IOException;
+
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/record/RecordsIterator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/RecordsIterator.java b/clients/src/main/java/org/apache/kafka/common/record/RecordsIterator.java
new file mode 100644
index 0000000..1bc8a65
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/record/RecordsIterator.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+package org.apache.kafka.common.record;
+
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.utils.AbstractIterator;
+import org.apache.kafka.common.utils.Utils;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+
+public class RecordsIterator extends AbstractIterator<LogEntry> {
+ private final LogInputStream logStream;
+ private final boolean shallow;
+ private DeepRecordsIterator innerIter;
+
+ public RecordsIterator(LogInputStream logStream, boolean shallow) {
+ this.logStream = logStream;
+ this.shallow = shallow;
+ }
+
+ /*
+ * Read the next record from the buffer.
+ *
+ * Note that in the compressed message set, each message value size is set as the size of the un-compressed
+ * version of the message value, so when we do de-compression allocating an array of the specified size for
+ * reading compressed value data is sufficient.
+ */
+ @Override
+ protected LogEntry makeNext() {
+ if (innerDone()) {
+ try {
+ LogEntry entry = logStream.nextEntry();
+ // No more record to return.
+ if (entry == null)
+ return allDone();
+
+ // decide whether to go shallow or deep iteration if it is compressed
+ CompressionType compressionType = entry.record().compressionType();
+ if (compressionType == CompressionType.NONE || shallow) {
+ return entry;
+ } else {
+ // init the inner iterator with the value payload of the message,
+ // which will de-compress the payload to a set of messages;
+ // since we assume nested compression is not allowed, the deep iterator
+ // would not try to further decompress underlying messages
+ // There will be at least one element in the inner iterator, so we don't
+ // need to call hasNext() here.
+ innerIter = new DeepRecordsIterator(entry);
+ return innerIter.next();
+ }
+ } catch (EOFException e) {
+ return allDone();
+ } catch (IOException e) {
+ throw new KafkaException(e);
+ }
+ } else {
+ return innerIter.next();
+ }
+ }
+
+ private boolean innerDone() {
+ return innerIter == null || !innerIter.hasNext();
+ }
+
+ private static class DataLogInputStream implements LogInputStream {
+ private final DataInputStream stream;
+
+ private DataLogInputStream(DataInputStream stream) {
+ this.stream = stream;
+ }
+
+ public LogEntry nextEntry() throws IOException {
+ long offset = stream.readLong();
+ int size = stream.readInt();
+ if (size < 0)
+ throw new IllegalStateException("Record with size " + size);
+
+ byte[] recordBuffer = new byte[size];
+ stream.readFully(recordBuffer, 0, size);
+ ByteBuffer buf = ByteBuffer.wrap(recordBuffer);
+ return new LogEntry(offset, new Record(buf));
+ }
+ }
+
+ private static class DeepRecordsIterator extends AbstractIterator<LogEntry> {
+ private final ArrayDeque<LogEntry> logEntries;
+ private final long absoluteBaseOffset;
+
+ private DeepRecordsIterator(LogEntry entry) {
+ CompressionType compressionType = entry.record().compressionType();
+ ByteBuffer buffer = entry.record().value();
+ DataInputStream stream = Compressor.wrapForInput(new ByteBufferInputStream(buffer), compressionType, entry.record().magic());
+ LogInputStream logStream = new DataLogInputStream(stream);
+
+ long wrapperRecordOffset = entry.offset();
+ long wrapperRecordTimestamp = entry.record().timestamp();
+ this.logEntries = new ArrayDeque<>();
+
+ // If relative offset is used, we need to decompress the entire message first to compute
+ // the absolute offset. For simplicity and because it's a format that is on its way out, we
+ // do the same for message format version 0
+ try {
+ while (true) {
+ try {
+ LogEntry logEntry = logStream.nextEntry();
+ if (entry.record().magic() > Record.MAGIC_VALUE_V0) {
+ Record recordWithTimestamp = new Record(
+ logEntry.record().buffer(),
+ wrapperRecordTimestamp,
+ entry.record().timestampType()
+ );
+ logEntry = new LogEntry(logEntry.offset(), recordWithTimestamp);
+ }
+ logEntries.add(logEntry);
+ } catch (EOFException e) {
+ break;
+ }
+ }
+ if (entry.record().magic() > Record.MAGIC_VALUE_V0)
+ this.absoluteBaseOffset = wrapperRecordOffset - logEntries.getLast().offset();
+ else
+ this.absoluteBaseOffset = -1;
+ } catch (IOException e) {
+ throw new KafkaException(e);
+ } finally {
+ Utils.closeQuietly(stream, "records iterator stream");
+ }
+ }
+
+ @Override
+ protected LogEntry makeNext() {
+ if (logEntries.isEmpty())
+ return allDone();
+
+ LogEntry entry = logEntries.remove();
+
+ // Convert offset to absolute offset if needed.
+ if (absoluteBaseOffset >= 0) {
+ long absoluteOffset = absoluteBaseOffset + entry.offset();
+ entry = new LogEntry(absoluteOffset, entry.record());
+ }
+
+ // decide whether to go shallow or deep iteration if it is compressed
+ CompressionType compression = entry.record().compressionType();
+ if (compression != CompressionType.NONE)
+ throw new InvalidRecordException("Inner messages must not be compressed");
+
+ return entry;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
index e6febe5..80182e6 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java
@@ -16,6 +16,8 @@
*/
package org.apache.kafka.common.requests;
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.types.Struct;
@@ -27,10 +29,14 @@ public abstract class AbstractRequest extends AbstractRequestResponse {
super(struct);
}
+ public Send toSend(String destination, RequestHeader header) {
+ return new NetworkSend(destination, serialize(header, this));
+ }
+
/**
* Get an error response for a request for a given api version
*/
- public abstract AbstractRequestResponse getErrorResponse(int versionId, Throwable e);
+ public abstract AbstractResponse getErrorResponse(int versionId, Throwable e);
/**
* Factory method for getting a request object based on ApiKey ID and a buffer
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
index 37aff6c..3ad16a5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java
@@ -19,7 +19,6 @@ import java.nio.ByteBuffer;
public abstract class AbstractRequestResponse {
protected final Struct struct;
-
public AbstractRequestResponse(Struct struct) {
this.struct = struct;
}
@@ -63,4 +62,12 @@ public abstract class AbstractRequestResponse {
AbstractRequestResponse other = (AbstractRequestResponse) obj;
return struct.equals(other.struct);
}
+
+ public static ByteBuffer serialize(AbstractRequestResponse header, AbstractRequestResponse body) {
+ ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + body.sizeOf());
+ header.writeTo(buffer);
+ body.writeTo(buffer);
+ buffer.rewind();
+ return buffer;
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
new file mode 100644
index 0000000..8bbc25a
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Send;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.types.Struct;
+
+public abstract class AbstractResponse extends AbstractRequestResponse {
+
+ public AbstractResponse(Struct struct) {
+ super(struct);
+ }
+
+ public Send toSend(String destination, RequestHeader request) {
+ ResponseHeader responseHeader = new ResponseHeader(request.correlationId());
+ return new NetworkSend(destination, serialize(responseHeader, this));
+ }
+
+ public static AbstractResponse getResponse(int requestId, Struct struct) {
+ ApiKeys apiKey = ApiKeys.forId(requestId);
+ switch (apiKey) {
+ case PRODUCE:
+ return new ProduceResponse(struct);
+ case FETCH:
+ return new FetchResponse(struct);
+ case LIST_OFFSETS:
+ return new ListOffsetResponse(struct);
+ case METADATA:
+ return new MetadataResponse(struct);
+ case OFFSET_COMMIT:
+ return new OffsetCommitResponse(struct);
+ case OFFSET_FETCH:
+ return new OffsetFetchResponse(struct);
+ case GROUP_COORDINATOR:
+ return new GroupCoordinatorResponse(struct);
+ case JOIN_GROUP:
+ return new JoinGroupResponse(struct);
+ case HEARTBEAT:
+ return new HeartbeatResponse(struct);
+ case LEAVE_GROUP:
+ return new LeaveGroupResponse(struct);
+ case SYNC_GROUP:
+ return new SyncGroupResponse(struct);
+ case STOP_REPLICA:
+ return new StopReplicaResponse(struct);
+ case CONTROLLED_SHUTDOWN_KEY:
+ return new ControlledShutdownResponse(struct);
+ case UPDATE_METADATA_KEY:
+ return new UpdateMetadataResponse(struct);
+ case LEADER_AND_ISR:
+ return new LeaderAndIsrResponse(struct);
+ case DESCRIBE_GROUPS:
+ return new DescribeGroupsResponse(struct);
+ case LIST_GROUPS:
+ return new ListGroupsResponse(struct);
+ case SASL_HANDSHAKE:
+ return new SaslHandshakeResponse(struct);
+ case API_VERSIONS:
+ return new ApiVersionsResponse(struct);
+ case CREATE_TOPICS:
+ return new CreateTopicsResponse(struct);
+ case DELETE_TOPICS:
+ return new DeleteTopicsResponse(struct);
+ default:
+ throw new AssertionError(String.format("ApiKey %s is not currently handled in `getResponse`, the " +
+ "code should be updated to do so.", apiKey));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
index b78c759..d9ef37e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java
@@ -34,7 +34,7 @@ public class ApiVersionsRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
short errorCode = Errors.forException(e).code();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
index fe995b2..0bf1039 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java
@@ -27,7 +27,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class ApiVersionsResponse extends AbstractRequestResponse {
+public class ApiVersionsResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.API_VERSIONS.id);
private static final ApiVersionsResponse API_VERSIONS_RESPONSE = createApiVersionsResponse();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
index 9ac127d..c2ace32 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java
@@ -42,7 +42,7 @@ public class ControlledShutdownRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
throw new IllegalArgumentException(String.format("Version 0 is not supported. It is only supported by " +
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
index 862264f..1996f82 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java
@@ -24,7 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-public class ControlledShutdownResponse extends AbstractRequestResponse {
+public class ControlledShutdownResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.CONTROLLED_SHUTDOWN_KEY.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
index 3977835..7c440dd 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java
@@ -194,7 +194,7 @@ public class CreateTopicsRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<String, Errors> topicErrors = new HashMap<>();
for (String topic : topics.keySet()) {
topicErrors.put(topic, Errors.forException(e));
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java
index da8c3ce..303b779 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class CreateTopicsResponse extends AbstractRequestResponse {
+public class CreateTopicsResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.CREATE_TOPICS.id);
private static final String TOPIC_ERROR_CODES_KEY_NAME = "topic_error_codes";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
index f78c428..0632cc0 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java
@@ -58,7 +58,7 @@ public class DeleteTopicsRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<String, Errors> topicErrors = new HashMap<>();
for (String topic : topics)
topicErrors.put(topic, Errors.forException(e));
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
index e474feb..ed6a63d 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class DeleteTopicsResponse extends AbstractRequestResponse {
+public class DeleteTopicsResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.DELETE_TOPICS.id);
private static final String TOPIC_ERROR_CODES_KEY_NAME = "topic_error_codes";
private static final String TOPIC_KEY_NAME = "topic";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
index a870b8f..b965c91 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java
@@ -46,7 +46,7 @@ public class DescribeGroupsRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
return DescribeGroupsResponse.fromError(Errors.forException(e), groupIds);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
index 2d4faee..2eff628 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java
@@ -25,7 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class DescribeGroupsResponse extends AbstractRequestResponse {
+public class DescribeGroupsResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.DESCRIBE_GROUPS.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
index dcdfd9c..fd4c747 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
@@ -24,6 +24,7 @@ import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.MemoryRecords;
public class FetchRequest extends AbstractRequest {
@@ -80,7 +81,7 @@ public class FetchRequest extends AbstractRequest {
int partition = topicEntry.getKey().partition();
T partitionData = topicEntry.getValue();
if (topics.isEmpty() || !topics.get(topics.size() - 1).topic.equals(topic))
- topics.add(new TopicAndPartitionData(topic));
+ topics.add(new TopicAndPartitionData<T>(topic));
topics.get(topics.size() - 1).partitions.put(partition, partitionData);
}
return topics;
@@ -131,11 +132,11 @@ public class FetchRequest extends AbstractRequest {
struct.set(MIN_BYTES_KEY_NAME, minBytes);
if (version >= 3)
struct.set(MAX_BYTES_KEY_NAME, maxBytes);
- List<Struct> topicArray = new ArrayList<Struct>();
+ List<Struct> topicArray = new ArrayList<>();
for (TopicAndPartitionData<PartitionData> topicEntry : topicsData) {
Struct topicData = struct.instance(TOPICS_KEY_NAME);
topicData.set(TOPIC_KEY_NAME, topicEntry.topic);
- List<Struct> partitionArray = new ArrayList<Struct>();
+ List<Struct> partitionArray = new ArrayList<>();
for (Map.Entry<Integer, PartitionData> partitionEntry : topicEntry.partitions.entrySet()) {
PartitionData fetchPartitionData = partitionEntry.getValue();
Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
@@ -180,13 +181,13 @@ public class FetchRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<TopicPartition, FetchResponse.PartitionData> responseData = new LinkedHashMap<>();
for (Map.Entry<TopicPartition, PartitionData> entry: fetchData.entrySet()) {
FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData(Errors.forException(e).code(),
- FetchResponse.INVALID_HIGHWATERMARK,
- FetchResponse.EMPTY_RECORD_SET);
+ FetchResponse.INVALID_HIGHWATERMARK, MemoryRecords.EMPTY);
+
responseData.put(entry.getKey(), partitionResponse);
}
@@ -223,6 +224,10 @@ public class FetchRequest extends AbstractRequest {
return fetchData;
}
+ public boolean isFromFollower() {
+ return replicaId >= 0;
+ }
+
public static FetchRequest parse(ByteBuffer buffer, int versionId) {
return new FetchRequest(ProtoUtils.parseRequest(ApiKeys.FETCH.id, versionId, buffer));
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
index 111d197..ec2ab47 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java
@@ -17,10 +17,15 @@
package org.apache.kafka.common.requests;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.network.ByteBufferSend;
+import org.apache.kafka.common.network.MultiSend;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.protocol.types.Type;
+import org.apache.kafka.common.record.Records;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -29,9 +34,9 @@ import java.util.List;
import java.util.Map;
/**
- * This wrapper supports both v0 and v1 of FetchResponse.
+ * This wrapper supports all versions of the Fetch API
*/
-public class FetchResponse extends AbstractRequestResponse {
+public class FetchResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.FETCH.id);
private static final String RESPONSES_KEY_NAME = "responses";
@@ -42,14 +47,15 @@ public class FetchResponse extends AbstractRequestResponse {
private static final String THROTTLE_TIME_KEY_NAME = "throttle_time_ms";
// partition level field names
+ private static final String PARTITION_HEADER_KEY_NAME = "partition_header";
private static final String PARTITION_KEY_NAME = "partition";
private static final String ERROR_CODE_KEY_NAME = "error_code";
// Default throttle time
private static final int DEFAULT_THROTTLE_TIME = 0;
- /**
- * Possible error code:
+ /**
+ * Possible error codes:
*
* OFFSET_OUT_OF_RANGE (1)
* UNKNOWN_TOPIC_OR_PARTITION (3)
@@ -62,7 +68,6 @@ public class FetchResponse extends AbstractRequestResponse {
private static final String RECORD_SET_KEY_NAME = "record_set";
public static final long INVALID_HIGHWATERMARK = -1L;
- public static final ByteBuffer EMPTY_RECORD_SET = ByteBuffer.allocate(0);
private final LinkedHashMap<TopicPartition, PartitionData> responseData;
private final int throttleTime;
@@ -70,12 +75,12 @@ public class FetchResponse extends AbstractRequestResponse {
public static final class PartitionData {
public final short errorCode;
public final long highWatermark;
- public final ByteBuffer recordSet;
+ public final Records records;
- public PartitionData(short errorCode, long highWatermark, ByteBuffer recordSet) {
+ public PartitionData(short errorCode, long highWatermark, Records records) {
this.errorCode = errorCode;
this.highWatermark = highWatermark;
- this.recordSet = recordSet;
+ this.records = records;
}
}
@@ -106,32 +111,9 @@ public class FetchResponse extends AbstractRequestResponse {
this(3, responseData, throttleTime);
}
- private FetchResponse(int version, LinkedHashMap<TopicPartition, PartitionData> responseData, int throttleTime) {
+ public FetchResponse(int version, LinkedHashMap<TopicPartition, PartitionData> responseData, int throttleTime) {
super(new Struct(ProtoUtils.responseSchema(ApiKeys.FETCH.id, version)));
-
- List<FetchRequest.TopicAndPartitionData<PartitionData>> topicsData = FetchRequest.TopicAndPartitionData.batchByTopic(responseData);
- List<Struct> topicArray = new ArrayList<>();
- for (FetchRequest.TopicAndPartitionData<PartitionData> topicEntry: topicsData) {
- Struct topicData = struct.instance(RESPONSES_KEY_NAME);
- topicData.set(TOPIC_KEY_NAME, topicEntry.topic);
- List<Struct> partitionArray = new ArrayList<>();
- for (Map.Entry<Integer, PartitionData> partitionEntry : topicEntry.partitions.entrySet()) {
- PartitionData fetchPartitionData = partitionEntry.getValue();
- Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
- partitionData.set(PARTITION_KEY_NAME, partitionEntry.getKey());
- partitionData.set(ERROR_CODE_KEY_NAME, fetchPartitionData.errorCode);
- partitionData.set(HIGH_WATERMARK_KEY_NAME, fetchPartitionData.highWatermark);
- partitionData.set(RECORD_SET_KEY_NAME, fetchPartitionData.recordSet);
- partitionArray.add(partitionData);
- }
- topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
- topicArray.add(topicData);
- }
- struct.set(RESPONSES_KEY_NAME, topicArray.toArray());
-
- if (version >= 1)
- struct.set(THROTTLE_TIME_KEY_NAME, throttleTime);
-
+ writeStruct(struct, version, responseData, throttleTime);
this.responseData = responseData;
this.throttleTime = throttleTime;
}
@@ -144,11 +126,12 @@ public class FetchResponse extends AbstractRequestResponse {
String topic = topicResponse.getString(TOPIC_KEY_NAME);
for (Object partitionResponseObj : topicResponse.getArray(PARTITIONS_KEY_NAME)) {
Struct partitionResponse = (Struct) partitionResponseObj;
- int partition = partitionResponse.getInt(PARTITION_KEY_NAME);
- short errorCode = partitionResponse.getShort(ERROR_CODE_KEY_NAME);
- long highWatermark = partitionResponse.getLong(HIGH_WATERMARK_KEY_NAME);
- ByteBuffer recordSet = partitionResponse.getBytes(RECORD_SET_KEY_NAME);
- PartitionData partitionData = new PartitionData(errorCode, highWatermark, recordSet);
+ Struct partitionResponseHeader = partitionResponse.getStruct(PARTITION_HEADER_KEY_NAME);
+ int partition = partitionResponseHeader.getInt(PARTITION_KEY_NAME);
+ short errorCode = partitionResponseHeader.getShort(ERROR_CODE_KEY_NAME);
+ long highWatermark = partitionResponseHeader.getLong(HIGH_WATERMARK_KEY_NAME);
+ Records records = partitionResponse.getRecords(RECORD_SET_KEY_NAME);
+ PartitionData partitionData = new PartitionData(errorCode, highWatermark, records);
responseData.put(new TopicPartition(topic, partition), partitionData);
}
}
@@ -156,6 +139,22 @@ public class FetchResponse extends AbstractRequestResponse {
this.throttleTime = struct.hasField(THROTTLE_TIME_KEY_NAME) ? struct.getInt(THROTTLE_TIME_KEY_NAME) : DEFAULT_THROTTLE_TIME;
}
+ @Override
+ public Send toSend(String dest, RequestHeader requestHeader) {
+ ResponseHeader responseHeader = new ResponseHeader(requestHeader.correlationId());
+
+ // write the total size and the response header
+ ByteBuffer buffer = ByteBuffer.allocate(responseHeader.sizeOf() + 4);
+ buffer.putInt(responseHeader.sizeOf() + struct.sizeOf());
+ responseHeader.writeTo(buffer);
+ buffer.rewind();
+
+ List<Send> sends = new ArrayList<>();
+ sends.add(new ByteBufferSend(dest, buffer));
+ addResponseData(dest, sends);
+ return new MultiSend(dest, sends);
+ }
+
public LinkedHashMap<TopicPartition, PartitionData> responseData() {
return responseData;
}
@@ -171,4 +170,92 @@ public class FetchResponse extends AbstractRequestResponse {
public static FetchResponse parse(ByteBuffer buffer, int version) {
return new FetchResponse(ProtoUtils.responseSchema(ApiKeys.FETCH.id, version).read(buffer));
}
+
+ private void addResponseData(String dest, List<Send> sends) {
+ Object[] allTopicData = struct.getArray(RESPONSES_KEY_NAME);
+
+ if (struct.hasField(THROTTLE_TIME_KEY_NAME)) {
+ int throttleTime = struct.getInt(THROTTLE_TIME_KEY_NAME);
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ buffer.putInt(throttleTime);
+ buffer.putInt(allTopicData.length);
+ buffer.rewind();
+ sends.add(new ByteBufferSend(dest, buffer));
+ } else {
+ ByteBuffer buffer = ByteBuffer.allocate(4);
+ buffer.putInt(allTopicData.length);
+ buffer.rewind();
+ sends.add(new ByteBufferSend(dest, buffer));
+ }
+
+ for (Object topicData : allTopicData)
+ addTopicData(dest, sends, (Struct) topicData);
+ }
+
+ private void addTopicData(String dest, List<Send> sends, Struct topicData) {
+ String topic = topicData.getString(TOPIC_KEY_NAME);
+ Object[] allPartitionData = topicData.getArray(PARTITIONS_KEY_NAME);
+
+ // include the topic header and the count for the number of partitions
+ ByteBuffer buffer = ByteBuffer.allocate(Type.STRING.sizeOf(topic) + 4);
+ Type.STRING.write(buffer, topic);
+ buffer.putInt(allPartitionData.length);
+ buffer.rewind();
+ sends.add(new ByteBufferSend(dest, buffer));
+
+ for (Object partitionData : allPartitionData)
+ addPartitionData(dest, sends, (Struct) partitionData);
+ }
+
+ private void addPartitionData(String dest, List<Send> sends, Struct partitionData) {
+ Struct header = partitionData.getStruct(PARTITION_HEADER_KEY_NAME);
+ Records records = partitionData.getRecords(RECORD_SET_KEY_NAME);
+
+ // include the partition header and the size of the record set
+ ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + 4);
+ header.writeTo(buffer);
+ buffer.putInt(records.sizeInBytes());
+ buffer.rewind();
+ sends.add(new ByteBufferSend(dest, buffer));
+
+ // finally the send for the record set itself
+ sends.add(new RecordsSend(dest, records));
+ }
+
+ private static void writeStruct(Struct struct,
+ int version,
+ LinkedHashMap<TopicPartition, PartitionData> responseData,
+ int throttleTime) {
+ List<FetchRequest.TopicAndPartitionData<PartitionData>> topicsData = FetchRequest.TopicAndPartitionData.batchByTopic(responseData);
+ List<Struct> topicArray = new ArrayList<>();
+ for (FetchRequest.TopicAndPartitionData<PartitionData> topicEntry: topicsData) {
+ Struct topicData = struct.instance(RESPONSES_KEY_NAME);
+ topicData.set(TOPIC_KEY_NAME, topicEntry.topic);
+ List<Struct> partitionArray = new ArrayList<>();
+ for (Map.Entry<Integer, PartitionData> partitionEntry : topicEntry.partitions.entrySet()) {
+ PartitionData fetchPartitionData = partitionEntry.getValue();
+ Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
+ Struct partitionDataHeader = partitionData.instance(PARTITION_HEADER_KEY_NAME);
+ partitionDataHeader.set(PARTITION_KEY_NAME, partitionEntry.getKey());
+ partitionDataHeader.set(ERROR_CODE_KEY_NAME, fetchPartitionData.errorCode);
+ partitionDataHeader.set(HIGH_WATERMARK_KEY_NAME, fetchPartitionData.highWatermark);
+ partitionData.set(PARTITION_HEADER_KEY_NAME, partitionDataHeader);
+ partitionData.set(RECORD_SET_KEY_NAME, fetchPartitionData.records);
+ partitionArray.add(partitionData);
+ }
+ topicData.set(PARTITIONS_KEY_NAME, partitionArray.toArray());
+ topicArray.add(topicData);
+ }
+ struct.set(RESPONSES_KEY_NAME, topicArray.toArray());
+
+ if (version >= 1)
+ struct.set(THROTTLE_TIME_KEY_NAME, throttleTime);
+ }
+
+ public static int sizeOf(int version, LinkedHashMap<TopicPartition, PartitionData> responseData) {
+ Struct struct = new Struct(ProtoUtils.responseSchema(ApiKeys.FETCH.id, version));
+ writeStruct(struct, version, responseData, 0);
+ return 4 + struct.sizeOf();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorRequest.java
index 0b98e55..7fee476 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorRequest.java
@@ -41,7 +41,7 @@ public class GroupCoordinatorRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
return new GroupCoordinatorResponse(Errors.GROUP_COORDINATOR_NOT_AVAILABLE.code(), Node.noNode());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorResponse.java
index 8e7beb4..1f447f7 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/GroupCoordinatorResponse.java
@@ -20,7 +20,7 @@ import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-public class GroupCoordinatorResponse extends AbstractRequestResponse {
+public class GroupCoordinatorResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.GROUP_COORDINATOR.id);
private static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
index 02eaa99..3e7401c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java
@@ -49,7 +49,7 @@ public class HeartbeatRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
return new HeartbeatResponse(Errors.forException(e).code());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java
index 7fe227c..72f0175 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java
@@ -19,7 +19,7 @@ import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-public class HeartbeatResponse extends AbstractRequestResponse {
+public class HeartbeatResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.HEARTBEAT.id);
private static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
index 2845ee0..51855b6 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java
@@ -142,7 +142,7 @@ public class JoinGroupRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
case 1:
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
index 8895ace..e7fc5b1 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java
@@ -23,7 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class JoinGroupResponse extends AbstractRequestResponse {
+public class JoinGroupResponse extends AbstractResponse {
private static final short CURRENT_VERSION = ProtoUtils.latestVersion(ApiKeys.JOIN_GROUP.id);
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.JOIN_GROUP.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
index 52b9674..79dcd4a 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java
@@ -145,7 +145,7 @@ public class LeaderAndIsrRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<TopicPartition, Short> responses = new HashMap<>(partitionStates.size());
for (TopicPartition partition : partitionStates.keySet()) {
responses.put(partition, Errors.forException(e).code());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
index df57714..a754def 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java
@@ -26,7 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class LeaderAndIsrResponse extends AbstractRequestResponse {
+public class LeaderAndIsrResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.LEADER_AND_ISR.id);
private static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
index 3047193..6a3f8a6 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java
@@ -43,7 +43,7 @@ public class LeaveGroupRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
return new LeaveGroupResponse(Errors.forException(e).code());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
index 6481ca7..9c7998b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java
@@ -18,7 +18,7 @@ import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
-public class LeaveGroupResponse extends AbstractRequestResponse {
+public class LeaveGroupResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.LEAVE_GROUP.id);
private static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
index 3160702..3fd3b81 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java
@@ -34,7 +34,7 @@ public class ListGroupsRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
short errorCode = Errors.forException(e).code();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
index 5519670..98573f8 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-public class ListGroupsResponse extends AbstractRequestResponse {
+public class ListGroupsResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.LIST_GROUPS.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
index 1aed523..c1db82d 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java
@@ -166,7 +166,7 @@ public class ListOffsetRequest extends AbstractRequest {
@Override
@SuppressWarnings("deprecation")
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<TopicPartition, ListOffsetResponse.PartitionData> responseData = new HashMap<TopicPartition, ListOffsetResponse.PartitionData>();
if (versionId == 0) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
index dbeef05..0b257bc 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class ListOffsetResponse extends AbstractRequestResponse {
+public class ListOffsetResponse extends AbstractResponse {
public static final long UNKNOWN_TIMESTAMP = -1L;
public static final long UNKNOWN_OFFSET = -1L;
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
index 24a9bfc..f7d8f8b 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java
@@ -64,7 +64,7 @@ public class MetadataRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
List<MetadataResponse.TopicMetadata> topicMetadatas = new ArrayList<>();
Errors error = Errors.forException(e);
List<MetadataResponse.PartitionMetadata> partitions = Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
index 444941b..a8baee5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-public class MetadataResponse extends AbstractRequestResponse {
+public class MetadataResponse extends AbstractResponse {
private static final short CURRENT_VERSION = ProtoUtils.latestVersion(ApiKeys.METADATA.id);
private static final String BROKERS_KEY_NAME = "brokers";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
index df18486..1dfad1e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java
@@ -154,7 +154,7 @@ public class OffsetCommitRequest extends AbstractRequest {
for (Map.Entry<String, Map<Integer, PartitionData>> topicEntry: topicsData.entrySet()) {
Struct topicData = struct.instance(TOPICS_KEY_NAME);
topicData.set(TOPIC_KEY_NAME, topicEntry.getKey());
- List<Struct> partitionArray = new ArrayList<Struct>();
+ List<Struct> partitionArray = new ArrayList<>();
for (Map.Entry<Integer, PartitionData> partitionEntry : topicEntry.getValue().entrySet()) {
PartitionData fetchPartitionData = partitionEntry.getValue();
Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
@@ -194,7 +194,7 @@ public class OffsetCommitRequest extends AbstractRequest {
else
retentionTime = DEFAULT_RETENTION_TIME;
- offsetData = new HashMap<TopicPartition, PartitionData>();
+ offsetData = new HashMap<>();
for (Object topicDataObj : struct.getArray(TOPICS_KEY_NAME)) {
Struct topicData = (Struct) topicDataObj;
String topic = topicData.getString(TOPIC_KEY_NAME);
@@ -217,8 +217,8 @@ public class OffsetCommitRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
- Map<TopicPartition, Short> responseData = new HashMap<TopicPartition, Short>();
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
+ Map<TopicPartition, Short> responseData = new HashMap<>();
for (Map.Entry<TopicPartition, PartitionData> entry: offsetData.entrySet()) {
responseData.put(entry.getKey(), Errors.forException(e).code());
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
index 1dfda93..abb260e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java
@@ -25,7 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class OffsetCommitResponse extends AbstractRequestResponse {
+public class OffsetCommitResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.OFFSET_COMMIT.id);
private static final String RESPONSES_KEY_NAME = "responses";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
index 422328e..ede0f27 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java
@@ -51,11 +51,11 @@ public class OffsetFetchRequest extends AbstractRequest {
Map<String, List<Integer>> topicsData = CollectionUtils.groupDataByTopic(partitions);
struct.set(GROUP_ID_KEY_NAME, groupId);
- List<Struct> topicArray = new ArrayList<Struct>();
+ List<Struct> topicArray = new ArrayList<>();
for (Map.Entry<String, List<Integer>> entries: topicsData.entrySet()) {
Struct topicData = struct.instance(TOPICS_KEY_NAME);
topicData.set(TOPIC_KEY_NAME, entries.getKey());
- List<Struct> partitionArray = new ArrayList<Struct>();
+ List<Struct> partitionArray = new ArrayList<>();
for (Integer partiitonId : entries.getValue()) {
Struct partitionData = topicData.instance(PARTITIONS_KEY_NAME);
partitionData.set(PARTITION_KEY_NAME, partiitonId);
@@ -71,7 +71,7 @@ public class OffsetFetchRequest extends AbstractRequest {
public OffsetFetchRequest(Struct struct) {
super(struct);
- partitions = new ArrayList<TopicPartition>();
+ partitions = new ArrayList<>();
for (Object topicResponseObj : struct.getArray(TOPICS_KEY_NAME)) {
Struct topicResponse = (Struct) topicResponseObj;
String topic = topicResponse.getString(TOPIC_KEY_NAME);
@@ -85,8 +85,8 @@ public class OffsetFetchRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
- Map<TopicPartition, OffsetFetchResponse.PartitionData> responseData = new HashMap<TopicPartition, OffsetFetchResponse.PartitionData>();
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
+ Map<TopicPartition, OffsetFetchResponse.PartitionData> responseData = new HashMap<>();
for (TopicPartition partition: partitions) {
responseData.put(partition, new OffsetFetchResponse.PartitionData(OffsetFetchResponse.INVALID_OFFSET,
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
index 1715777..ae4e066 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java
@@ -26,7 +26,7 @@ import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.utils.CollectionUtils;
-public class OffsetFetchResponse extends AbstractRequestResponse {
+public class OffsetFetchResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.OFFSET_FETCH.id);
private static final String RESPONSES_KEY_NAME = "responses";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
index c7d41e6..25209ce 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
@@ -19,6 +19,7 @@ import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Schema;
import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.utils.CollectionUtils;
@@ -45,23 +46,23 @@ public class ProduceRequest extends AbstractRequest {
private final short acks;
private final int timeout;
- private final Map<TopicPartition, ByteBuffer> partitionRecords;
+ private final Map<TopicPartition, MemoryRecords> partitionRecords;
- public ProduceRequest(short acks, int timeout, Map<TopicPartition, ByteBuffer> partitionRecords) {
+ public ProduceRequest(short acks, int timeout, Map<TopicPartition, MemoryRecords> partitionRecords) {
super(new Struct(CURRENT_SCHEMA));
- Map<String, Map<Integer, ByteBuffer>> recordsByTopic = CollectionUtils.groupDataByTopic(partitionRecords);
+ Map<String, Map<Integer, MemoryRecords>> recordsByTopic = CollectionUtils.groupDataByTopic(partitionRecords);
struct.set(ACKS_KEY_NAME, acks);
struct.set(TIMEOUT_KEY_NAME, timeout);
- List<Struct> topicDatas = new ArrayList<Struct>(recordsByTopic.size());
- for (Map.Entry<String, Map<Integer, ByteBuffer>> entry : recordsByTopic.entrySet()) {
+ List<Struct> topicDatas = new ArrayList<>(recordsByTopic.size());
+ for (Map.Entry<String, Map<Integer, MemoryRecords>> entry : recordsByTopic.entrySet()) {
Struct topicData = struct.instance(TOPIC_DATA_KEY_NAME);
topicData.set(TOPIC_KEY_NAME, entry.getKey());
- List<Struct> partitionArray = new ArrayList<Struct>();
- for (Map.Entry<Integer, ByteBuffer> partitionEntry : entry.getValue().entrySet()) {
- ByteBuffer buffer = partitionEntry.getValue().duplicate();
+ List<Struct> partitionArray = new ArrayList<>();
+ for (Map.Entry<Integer, MemoryRecords> partitionEntry : entry.getValue().entrySet()) {
+ MemoryRecords records = partitionEntry.getValue();
Struct part = topicData.instance(PARTITION_DATA_KEY_NAME)
.set(PARTITION_KEY_NAME, partitionEntry.getKey())
- .set(RECORD_SET_KEY_NAME, buffer);
+ .set(RECORD_SET_KEY_NAME, records);
partitionArray.add(part);
}
topicData.set(PARTITION_DATA_KEY_NAME, partitionArray.toArray());
@@ -75,14 +76,14 @@ public class ProduceRequest extends AbstractRequest {
public ProduceRequest(Struct struct) {
super(struct);
- partitionRecords = new HashMap<TopicPartition, ByteBuffer>();
+ partitionRecords = new HashMap<>();
for (Object topicDataObj : struct.getArray(TOPIC_DATA_KEY_NAME)) {
Struct topicData = (Struct) topicDataObj;
String topic = topicData.getString(TOPIC_KEY_NAME);
for (Object partitionResponseObj : topicData.getArray(PARTITION_DATA_KEY_NAME)) {
Struct partitionResponse = (Struct) partitionResponseObj;
int partition = partitionResponse.getInt(PARTITION_KEY_NAME);
- ByteBuffer records = partitionResponse.getBytes(RECORD_SET_KEY_NAME);
+ MemoryRecords records = (MemoryRecords) partitionResponse.getRecords(RECORD_SET_KEY_NAME);
partitionRecords.put(new TopicPartition(topic, partition), records);
}
}
@@ -91,14 +92,14 @@ public class ProduceRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
/* In case the producer doesn't actually want any response */
if (acks == 0)
return null;
- Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<TopicPartition, ProduceResponse.PartitionResponse>();
+ Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
- for (Map.Entry<TopicPartition, ByteBuffer> entry : partitionRecords.entrySet()) {
+ for (Map.Entry<TopicPartition, MemoryRecords> entry : partitionRecords.entrySet()) {
responseMap.put(entry.getKey(), new ProduceResponse.PartitionResponse(Errors.forException(e).code(), ProduceResponse.INVALID_OFFSET, Record.NO_TIMESTAMP));
}
@@ -122,7 +123,7 @@ public class ProduceRequest extends AbstractRequest {
return timeout;
}
- public Map<TopicPartition, ByteBuffer> partitionRecords() {
+ public Map<TopicPartition, MemoryRecords> partitionRecords() {
return partitionRecords;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
index 58175e1..71e6ab5 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java
@@ -28,7 +28,7 @@ import java.util.Map;
/**
* This wrapper supports both v0 and v1 of ProduceResponse.
*/
-public class ProduceResponse extends AbstractRequestResponse {
+public class ProduceResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.PRODUCE.id);
private static final String RESPONSES_KEY_NAME = "responses";
@@ -98,7 +98,7 @@ public class ProduceResponse extends AbstractRequestResponse {
*/
public ProduceResponse(Struct struct) {
super(struct);
- responses = new HashMap<TopicPartition, PartitionResponse>();
+ responses = new HashMap<>();
for (Object topicResponse : struct.getArray(RESPONSES_KEY_NAME)) {
Struct topicRespStruct = (Struct) topicResponse;
String topic = topicRespStruct.getString(TOPIC_KEY_NAME);
@@ -117,11 +117,11 @@ public class ProduceResponse extends AbstractRequestResponse {
private void initCommonFields(Map<TopicPartition, PartitionResponse> responses) {
Map<String, Map<Integer, PartitionResponse>> responseByTopic = CollectionUtils.groupDataByTopic(responses);
- List<Struct> topicDatas = new ArrayList<Struct>(responseByTopic.size());
+ List<Struct> topicDatas = new ArrayList<>(responseByTopic.size());
for (Map.Entry<String, Map<Integer, PartitionResponse>> entry : responseByTopic.entrySet()) {
Struct topicData = struct.instance(RESPONSES_KEY_NAME);
topicData.set(TOPIC_KEY_NAME, entry.getKey());
- List<Struct> partitionArray = new ArrayList<Struct>();
+ List<Struct> partitionArray = new ArrayList<>();
for (Map.Entry<Integer, PartitionResponse> partitionEntry : entry.getValue().entrySet()) {
PartitionResponse part = partitionEntry.getValue();
Struct partStruct = topicData.instance(PARTITION_RESPONSES_KEY_NAME)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/RecordsSend.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RecordsSend.java b/clients/src/main/java/org/apache/kafka/common/requests/RecordsSend.java
new file mode 100644
index 0000000..1c3bb0d
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/requests/RecordsSend.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+package org.apache.kafka.common.requests;
+
+import org.apache.kafka.common.network.Send;
+import org.apache.kafka.common.network.TransportLayer;
+import org.apache.kafka.common.record.Records;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.GatheringByteChannel;
+
+public class RecordsSend implements Send {
+ private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocate(0);
+
+ private final String destination;
+ private final Records records;
+ private int remaining;
+ private boolean pending = false;
+
+ public RecordsSend(String destination, Records records) {
+ this.destination = destination;
+ this.records = records;
+ this.remaining = records.sizeInBytes();
+ }
+
+ @Override
+ public String destination() {
+ return destination;
+ }
+
+ @Override
+ public boolean completed() {
+ return remaining <= 0 && !pending;
+ }
+
+ @Override
+ public long writeTo(GatheringByteChannel channel) throws IOException {
+ long written = 0;
+ if (remaining > 0) {
+ written = records.writeTo(channel, size() - remaining, remaining);
+ if (written < 0)
+ throw new EOFException("Wrote negative bytes to channel. This shouldn't happen.");
+ }
+
+ if (channel instanceof TransportLayer) {
+ TransportLayer tl = (TransportLayer) channel;
+ pending = tl.hasPendingWrites();
+
+ if (remaining <= 0 && pending)
+ channel.write(EMPTY_BYTE_BUFFER);
+ }
+
+ remaining -= written;
+ return written;
+ }
+
+ @Override
+ public long size() {
+ return records.sizeInBytes();
+ }
+}
[3/5] kafka git commit: KAFKA-2066;
Use client-side FetchRequest/FetchResponse on server
Posted by jg...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java
deleted file mode 100644
index 02cac80..0000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
- * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
- * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kafka.common.network.NetworkSend;
-import org.apache.kafka.common.protocol.types.Struct;
-
-/**
- * A send object for a kafka request
- */
-public class RequestSend extends NetworkSend {
-
- private final RequestHeader header;
- private final Struct body;
-
- public RequestSend(String destination, RequestHeader header, Struct body) {
- super(destination, serialize(header, body));
- this.header = header;
- this.body = body;
- }
-
- public static ByteBuffer serialize(RequestHeader header, Struct body) {
- ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + body.sizeOf());
- header.writeTo(buffer);
- body.writeTo(buffer);
- buffer.rewind();
- return buffer;
- }
-
- public RequestHeader header() {
- return this.header;
- }
-
- public Struct body() {
- return body;
- }
-
- @Override
- public String toString() {
- return "RequestSend(header=" + header.toString() + ", body=" + body.toString() + ")";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java
deleted file mode 100644
index 9494de7..0000000
--- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.common.requests;
-
-import org.apache.kafka.common.network.NetworkSend;
-import org.apache.kafka.common.protocol.types.Struct;
-
-import java.nio.ByteBuffer;
-
-public class ResponseSend extends NetworkSend {
-
- public ResponseSend(String destination, ResponseHeader header, Struct body) {
- super(destination, serialize(header, body));
- }
-
- public ResponseSend(String destination, ResponseHeader header, AbstractRequestResponse response) {
- this(destination, header, response.toStruct());
- }
-
- public static ByteBuffer serialize(ResponseHeader header, Struct body) {
- ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + body.sizeOf());
- header.writeTo(buffer);
- body.writeTo(buffer);
- buffer.rewind();
- return buffer;
- }
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
index bddc9f0..97adcb3 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java
@@ -61,7 +61,7 @@ public class SaslHandshakeRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
List<String> enabledMechanisms = Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java
index c0fc495..6d7f734 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java
@@ -33,7 +33,7 @@ import org.apache.kafka.common.protocol.types.Struct;
* Response from SASL server which indicates if the client-chosen mechanism is enabled in the server.
* For error responses, the list of enabled mechanisms is included in the response.
*/
-public class SaslHandshakeResponse extends AbstractRequestResponse {
+public class SaslHandshakeResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.SASL_HANDSHAKE.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
index bc63521..e3e2507 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java
@@ -84,7 +84,7 @@ public class StopReplicaRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
Map<TopicPartition, Short> responses = new HashMap<>(partitions.size());
for (TopicPartition partition : partitions) {
responses.put(partition, Errors.forException(e).code());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
index 3c4db68..92d9e58 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java
@@ -26,7 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-public class StopReplicaResponse extends AbstractRequestResponse {
+public class StopReplicaResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.STOP_REPLICA.id);
private static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
index 606584b..efb484c 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java
@@ -83,7 +83,7 @@ public class SyncGroupRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
switch (versionId) {
case 0:
return new SyncGroupResponse(
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java
index 3584f14..f459656 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java
@@ -23,7 +23,7 @@ import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-public class SyncGroupResponse extends AbstractRequestResponse {
+public class SyncGroupResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.SYNC_GROUP.id);
public static final String ERROR_CODE_KEY_NAME = "error_code";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
index 1c21789..2e5ffb2 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java
@@ -242,7 +242,7 @@ public class UpdateMetadataRequest extends AbstractRequest {
}
@Override
- public AbstractRequestResponse getErrorResponse(int versionId, Throwable e) {
+ public AbstractResponse getErrorResponse(int versionId, Throwable e) {
if (versionId <= 2)
return new UpdateMetadataResponse(Errors.forException(e).code());
else
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
index 0ff35d9..e1c3634 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java
@@ -20,7 +20,7 @@ import org.apache.kafka.common.protocol.types.Struct;
import java.nio.ByteBuffer;
-public class UpdateMetadataResponse extends AbstractRequestResponse {
+public class UpdateMetadataResponse extends AbstractResponse {
private static final Schema CURRENT_SCHEMA = ProtoUtils.currentResponseSchema(ApiKeys.UPDATE_METADATA_KEY.id);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
index ba201dc..b9c3b33 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java
@@ -18,45 +18,44 @@
package org.apache.kafka.common.security.authenticator;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.util.Arrays;
-import java.util.Map;
-import java.security.Principal;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
-
-import javax.security.auth.Subject;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslClient;
-import javax.security.sasl.SaslException;
-
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.NetworkClient;
+import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.errors.IllegalSaslStateException;
import org.apache.kafka.common.errors.UnsupportedSaslMechanismException;
import org.apache.kafka.common.network.Authenticator;
import org.apache.kafka.common.network.Mode;
-import org.apache.kafka.common.network.NetworkSend;
import org.apache.kafka.common.network.NetworkReceive;
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.network.TransportLayer;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.types.SchemaException;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.RequestSend;
import org.apache.kafka.common.requests.SaslHandshakeRequest;
import org.apache.kafka.common.requests.SaslHandshakeResponse;
import org.apache.kafka.common.security.auth.AuthCallbackHandler;
import org.apache.kafka.common.security.auth.KafkaPrincipal;
import org.apache.kafka.common.security.auth.PrincipalBuilder;
-import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.NetworkClient;
-import org.apache.kafka.common.KafkaException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.security.auth.Subject;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.security.Principal;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.Map;
+
public class SaslClientAuthenticator implements Authenticator {
public enum SaslState {
@@ -81,7 +80,7 @@ public class SaslClientAuthenticator implements Authenticator {
// buffers used in `authenticate`
private NetworkReceive netInBuffer;
- private NetworkSend netOutBuffer;
+ private Send netOutBuffer;
// Current SASL state
private SaslState saslState;
@@ -156,7 +155,7 @@ public class SaslClientAuthenticator implements Authenticator {
String clientId = (String) configs.get(CommonClientConfigs.CLIENT_ID_CONFIG);
currentRequestHeader = new RequestHeader(ApiKeys.SASL_HANDSHAKE.id, clientId, correlationId++);
SaslHandshakeRequest handshakeRequest = new SaslHandshakeRequest(mechanism);
- send(RequestSend.serialize(currentRequestHeader, handshakeRequest.toStruct()));
+ send(handshakeRequest.toSend(node, currentRequestHeader));
setSaslState(SaslState.RECEIVE_HANDSHAKE_RESPONSE);
break;
case RECEIVE_HANDSHAKE_RESPONSE:
@@ -209,13 +208,13 @@ public class SaslClientAuthenticator implements Authenticator {
if (!saslClient.isComplete()) {
byte[] saslToken = createSaslToken(serverToken, isInitial);
if (saslToken != null)
- send(ByteBuffer.wrap(saslToken));
+ send(new NetworkSend(node, ByteBuffer.wrap(saslToken)));
}
}
- private void send(ByteBuffer buffer) throws IOException {
+ private void send(Send send) throws IOException {
try {
- netOutBuffer = new NetworkSend(node, buffer);
+ netOutBuffer = send;
flushNetOutBufferAndUpdateInterestOps();
} catch (IOException e) {
setSaslState(SaslState.FAILED);
@@ -281,7 +280,7 @@ public class SaslClientAuthenticator implements Authenticator {
// TODO: introspect about e: look for GSS information.
final String unknownServerErrorText =
"(Mechanism level: Server not found in Kerberos database (7) - UNKNOWN_SERVER)";
- if (e.toString().indexOf(unknownServerErrorText) > -1) {
+ if (e.toString().contains(unknownServerErrorText)) {
error += " This may be caused by Java's being unable to resolve the Kafka Broker's" +
" hostname correctly. You may want to try to adding" +
" '-Dsun.net.spi.nameservice.provider.1=dns,sun' to your client's JVMFLAGS environment." +
@@ -302,10 +301,10 @@ public class SaslClientAuthenticator implements Authenticator {
}
private void handleKafkaResponse(RequestHeader requestHeader, byte[] responseBytes) {
- Struct struct;
+ AbstractResponse response;
ApiKeys apiKey;
try {
- struct = NetworkClient.parseResponse(ByteBuffer.wrap(responseBytes), requestHeader);
+ response = NetworkClient.parseResponse(ByteBuffer.wrap(responseBytes), requestHeader);
apiKey = ApiKeys.forId(requestHeader.apiKey());
} catch (SchemaException | IllegalArgumentException e) {
LOG.debug("Invalid SASL mechanism response, server may be expecting only GSSAPI tokens");
@@ -313,7 +312,7 @@ public class SaslClientAuthenticator implements Authenticator {
}
switch (apiKey) {
case SASL_HANDSHAKE:
- handleSaslHandshakeResponse(new SaslHandshakeResponse(struct));
+ handleSaslHandshakeResponse((SaslHandshakeResponse) response);
break;
default:
throw new IllegalStateException("Unexpected API key during handshake: " + apiKey);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
index 206fe39..b193bf2 100644
--- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
+++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java
@@ -17,36 +17,7 @@
package org.apache.kafka.common.security.authenticator;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.security.Principal;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
-
-import javax.security.auth.login.Configuration;
-import javax.security.auth.Subject;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslServer;
-import javax.security.sasl.SaslException;
-
import org.apache.kafka.common.KafkaException;
-import org.apache.kafka.common.security.kerberos.KerberosName;
-import org.apache.kafka.common.security.kerberos.KerberosShortNamer;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.kafka.common.security.auth.AuthCallbackHandler;
-import org.apache.kafka.common.security.auth.KafkaPrincipal;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.errors.IllegalSaslStateException;
@@ -54,22 +25,49 @@ import org.apache.kafka.common.errors.UnsupportedSaslMechanismException;
import org.apache.kafka.common.errors.UnsupportedVersionException;
import org.apache.kafka.common.network.Authenticator;
import org.apache.kafka.common.network.Mode;
-import org.apache.kafka.common.network.NetworkSend;
import org.apache.kafka.common.network.NetworkReceive;
+import org.apache.kafka.common.network.NetworkSend;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.network.TransportLayer;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.Protocol;
import org.apache.kafka.common.protocol.types.SchemaException;
import org.apache.kafka.common.requests.AbstractRequest;
-import org.apache.kafka.common.requests.AbstractRequestResponse;
+import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.ApiVersionsResponse;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.ResponseHeader;
-import org.apache.kafka.common.requests.ResponseSend;
import org.apache.kafka.common.requests.SaslHandshakeRequest;
import org.apache.kafka.common.requests.SaslHandshakeResponse;
+import org.apache.kafka.common.security.auth.AuthCallbackHandler;
+import org.apache.kafka.common.security.auth.KafkaPrincipal;
import org.apache.kafka.common.security.auth.PrincipalBuilder;
+import org.apache.kafka.common.security.kerberos.KerberosName;
+import org.apache.kafka.common.security.kerberos.KerberosShortNamer;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+import org.ietf.jgss.Oid;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.Configuration;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.security.Principal;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class SaslServerAuthenticator implements Authenticator {
@@ -100,7 +98,7 @@ public class SaslServerAuthenticator implements Authenticator {
// buffers used in `authenticate`
private NetworkReceive netInBuffer;
- private NetworkSend netOutBuffer;
+ private Send netOutBuffer;
public SaslServerAuthenticator(String node, final Subject subject, KerberosShortNamer kerberosNameParser, String host, int maxReceiveSize) throws IOException {
if (subject == null)
@@ -364,9 +362,8 @@ public class SaslServerAuthenticator implements Authenticator {
sendKafkaResponse(requestHeader, ApiVersionsResponse.apiVersionsResponse());
}
- private void sendKafkaResponse(RequestHeader requestHeader, AbstractRequestResponse response) throws IOException {
- ResponseHeader responseHeader = new ResponseHeader(requestHeader.correlationId());
- netOutBuffer = new NetworkSend(node, ResponseSend.serialize(responseHeader, response.toStruct()));
+ private void sendKafkaResponse(RequestHeader requestHeader, AbstractResponse response) throws IOException {
+ netOutBuffer = response.toSend(node, requestHeader);
flushNetOutBufferAndUpdateInterestOps();
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/utils/AbstractIterator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/utils/AbstractIterator.java b/clients/src/main/java/org/apache/kafka/common/utils/AbstractIterator.java
index b798e73..d6771a3 100644
--- a/clients/src/main/java/org/apache/kafka/common/utils/AbstractIterator.java
+++ b/clients/src/main/java/org/apache/kafka/common/utils/AbstractIterator.java
@@ -25,9 +25,9 @@ import java.util.NoSuchElementException;
*/
public abstract class AbstractIterator<T> implements Iterator<T> {
- private static enum State {
+ private enum State {
READY, NOT_READY, DONE, FAILED
- };
+ }
private State state = State.NOT_READY;
private T next;
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/MockClient.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
index 0af4d34..9b445c7 100644
--- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java
+++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java
@@ -16,6 +16,14 @@
*/
package org.apache.kafka.clients;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.apache.kafka.common.utils.Time;
+
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.HashMap;
@@ -26,31 +34,24 @@ import java.util.Map;
import java.util.Queue;
import java.util.Set;
-import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.Node;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.utils.Time;
-
/**
* A mock network client for use testing code
*/
public class MockClient implements KafkaClient {
public static final RequestMatcher ALWAYS_TRUE = new RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
return true;
}
};
private class FutureResponse {
- public final Struct responseBody;
+ public final AbstractResponse responseBody;
public final boolean disconnected;
public final RequestMatcher requestMatcher;
public Node node;
- public FutureResponse(Struct responseBody, boolean disconnected, RequestMatcher requestMatcher, Node node) {
+ public FutureResponse(AbstractResponse responseBody, boolean disconnected, RequestMatcher requestMatcher, Node node) {
this.responseBody = responseBody;
this.disconnected = disconnected;
this.requestMatcher = requestMatcher;
@@ -125,8 +126,9 @@ public class MockClient implements KafkaClient {
Iterator<ClientRequest> iter = requests.iterator();
while (iter.hasNext()) {
ClientRequest request = iter.next();
- if (request.request().destination().equals(node)) {
- responses.add(new ClientResponse(request, now, true, null));
+ if (request.destination().equals(node)) {
+ responses.add(new ClientResponse(request.header(), request.callback(), request.destination(),
+ request.createdTimeMs(), now, true, null));
iter.remove();
}
}
@@ -138,19 +140,19 @@ public class MockClient implements KafkaClient {
Iterator<FutureResponse> iterator = futureResponses.iterator();
while (iterator.hasNext()) {
FutureResponse futureResp = iterator.next();
- if (futureResp.node != null && !request.request().destination().equals(futureResp.node.idString()))
+ if (futureResp.node != null && !request.destination().equals(futureResp.node.idString()))
continue;
- if (!futureResp.requestMatcher.matches(request))
+ if (!futureResp.requestMatcher.matches(request.body()))
throw new IllegalStateException("Next in line response did not match expected request");
- ClientResponse resp = new ClientResponse(request, time.milliseconds(), futureResp.disconnected, futureResp.responseBody);
+ ClientResponse resp = new ClientResponse(request.header(), request.callback(), request.destination(),
+ request.createdTimeMs(), time.milliseconds(), futureResp.disconnected, futureResp.responseBody);
responses.add(resp);
iterator.remove();
return;
}
- request.setSendTimeMs(now);
this.requests.add(request);
}
@@ -168,8 +170,7 @@ public class MockClient implements KafkaClient {
while (!this.responses.isEmpty()) {
ClientResponse response = this.responses.poll();
- if (response.request().hasCallback())
- response.request().callback().onComplete(response);
+ response.onComplete();
}
return copy;
@@ -179,75 +180,77 @@ public class MockClient implements KafkaClient {
return this.requests;
}
- public void respond(Struct body) {
- respond(body, false);
+ public void respond(AbstractResponse response) {
+ respond(response, false);
}
- public void respond(Struct body, boolean disconnected) {
+ public void respond(AbstractResponse response, boolean disconnected) {
ClientRequest request = requests.remove();
- responses.add(new ClientResponse(request, time.milliseconds(), disconnected, body));
+ responses.add(new ClientResponse(request.header(), request.callback(), request.destination(),
+ request.createdTimeMs(), time.milliseconds(), disconnected, response));
}
- public void respondFrom(Struct body, Node node) {
- respondFrom(body, node, false);
+ public void respondFrom(AbstractResponse response, Node node) {
+ respondFrom(response, node, false);
}
- public void respondFrom(Struct body, Node node, boolean disconnected) {
+ public void respondFrom(AbstractResponse response, Node node, boolean disconnected) {
Iterator<ClientRequest> iterator = requests.iterator();
while (iterator.hasNext()) {
ClientRequest request = iterator.next();
- if (request.request().destination().equals(node.idString())) {
+ if (request.destination().equals(node.idString())) {
iterator.remove();
- responses.add(new ClientResponse(request, time.milliseconds(), disconnected, body));
+ responses.add(new ClientResponse(request.header(), request.callback(), request.destination(),
+ request.createdTimeMs(), time.milliseconds(), disconnected, response));
return;
}
}
throw new IllegalArgumentException("No requests available to node " + node);
}
- public void prepareResponse(Struct body) {
- prepareResponse(ALWAYS_TRUE, body, false);
+ public void prepareResponse(AbstractResponse response) {
+ prepareResponse(ALWAYS_TRUE, response, false);
}
- public void prepareResponseFrom(Struct body, Node node) {
- prepareResponseFrom(ALWAYS_TRUE, body, node, false);
+ public void prepareResponseFrom(AbstractResponse response, Node node) {
+ prepareResponseFrom(ALWAYS_TRUE, response, node, false);
}
/**
* Prepare a response for a request matching the provided matcher. If the matcher does not
- * match, {@link #send(ClientRequest, long)} will throw IllegalStateException
+ * match, {@link KafkaClient#send(ClientRequest, long)} will throw IllegalStateException
* @param matcher The matcher to apply
- * @param body The response body
+ * @param response The response body
*/
- public void prepareResponse(RequestMatcher matcher, Struct body) {
- prepareResponse(matcher, body, false);
+ public void prepareResponse(RequestMatcher matcher, AbstractResponse response) {
+ prepareResponse(matcher, response, false);
}
- public void prepareResponseFrom(RequestMatcher matcher, Struct body, Node node) {
- prepareResponseFrom(matcher, body, node, false);
+ public void prepareResponseFrom(RequestMatcher matcher, AbstractResponse response, Node node) {
+ prepareResponseFrom(matcher, response, node, false);
}
- public void prepareResponse(Struct body, boolean disconnected) {
- prepareResponse(ALWAYS_TRUE, body, disconnected);
+ public void prepareResponse(AbstractResponse response, boolean disconnected) {
+ prepareResponse(ALWAYS_TRUE, response, disconnected);
}
- public void prepareResponseFrom(Struct body, Node node, boolean disconnected) {
- prepareResponseFrom(ALWAYS_TRUE, body, node, disconnected);
+ public void prepareResponseFrom(AbstractResponse response, Node node, boolean disconnected) {
+ prepareResponseFrom(ALWAYS_TRUE, response, node, disconnected);
}
/**
* Prepare a response for a request matching the provided matcher. If the matcher does not
- * match, {@link #send(ClientRequest, long)} will throw IllegalStateException
+ * match, {@link KafkaClient#send(ClientRequest, long)} will throw IllegalStateException
* @param matcher The matcher to apply
- * @param body The response body
+ * @param response The response body
* @param disconnected Whether the request was disconnected
*/
- public void prepareResponse(RequestMatcher matcher, Struct body, boolean disconnected) {
- prepareResponseFrom(matcher, body, null, disconnected);
+ public void prepareResponse(RequestMatcher matcher, AbstractResponse response, boolean disconnected) {
+ prepareResponseFrom(matcher, response, null, disconnected);
}
- public void prepareResponseFrom(RequestMatcher matcher, Struct body, Node node, boolean disconnected) {
- futureResponses.add(new FutureResponse(body, disconnected, matcher, node));
+ public void prepareResponseFrom(RequestMatcher matcher, AbstractResponse response, Node node, boolean disconnected) {
+ futureResponses.add(new FutureResponse(response, disconnected, matcher, node));
}
public void reset() {
@@ -307,12 +310,12 @@ public class MockClient implements KafkaClient {
/**
* The RequestMatcher provides a way to match a particular request to a response prepared
- * through {@link #prepareResponse(RequestMatcher, Struct)}. Basically this allows testers
+ * through {@link #prepareResponse(RequestMatcher, AbstractResponse)}. Basically this allows testers
* to inspect the request body for the type of the request or for specific fields that should be set,
* and to fail the test if it doesn't match.
*/
public interface RequestMatcher {
- boolean matches(ClientRequest request);
+ boolean matches(AbstractRequest body);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
index d305e8e..0966ee5 100644
--- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java
@@ -23,10 +23,10 @@ import org.apache.kafka.common.network.NetworkReceive;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.ProduceRequest;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.RequestSend;
import org.apache.kafka.common.requests.ResponseHeader;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.test.MockSelector;
@@ -66,10 +66,9 @@ public class NetworkClientTest {
@Test(expected = IllegalStateException.class)
public void testSendToUnreadyNode() {
- RequestSend send = new RequestSend("5",
- client.nextRequestHeader(ApiKeys.METADATA),
- new MetadataRequest(Arrays.asList("test")).toStruct());
- ClientRequest request = new ClientRequest(time.milliseconds(), false, send, null);
+ MetadataRequest metadataRequest = new MetadataRequest(Arrays.asList("test"));
+ RequestHeader header = client.nextRequestHeader(ApiKeys.METADATA);
+ ClientRequest request = new ClientRequest("5", time.milliseconds(), false, header, metadataRequest, null);
client.send(request, time.milliseconds());
client.poll(1, time.milliseconds());
}
@@ -91,10 +90,9 @@ public class NetworkClientTest {
client.poll(1, time.milliseconds());
assertTrue("The client should be ready", client.isReady(node, time.milliseconds()));
- ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, ByteBuffer>emptyMap());
+ ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, MemoryRecords>emptyMap());
RequestHeader reqHeader = client.nextRequestHeader(ApiKeys.PRODUCE);
- RequestSend send = new RequestSend(node.idString(), reqHeader, produceRequest.toStruct());
- ClientRequest request = new ClientRequest(time.milliseconds(), true, send, null);
+ ClientRequest request = new ClientRequest(node.idString(), time.milliseconds(), true, reqHeader, produceRequest, null);
client.send(request, time.milliseconds());
assertEquals("There should be 1 in-flight request after send", 1, client.inFlightRequestCount(node.idString()));
@@ -104,11 +102,10 @@ public class NetworkClientTest {
}
private void checkSimpleRequestResponse(NetworkClient networkClient) {
- ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, ByteBuffer>emptyMap());
+ ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, MemoryRecords>emptyMap());
RequestHeader reqHeader = networkClient.nextRequestHeader(ApiKeys.PRODUCE);
- RequestSend send = new RequestSend(node.idString(), reqHeader, produceRequest.toStruct());
TestCallbackHandler handler = new TestCallbackHandler();
- ClientRequest request = new ClientRequest(time.milliseconds(), true, send, handler);
+ ClientRequest request = new ClientRequest(node.idString(), time.milliseconds(), true, reqHeader, produceRequest, handler);
awaitReady(networkClient, node);
networkClient.send(request, time.milliseconds());
networkClient.poll(1, time.milliseconds());
@@ -126,7 +123,7 @@ public class NetworkClientTest {
assertEquals(1, responses.size());
assertTrue("The handler should have executed.", handler.executed);
assertTrue("Should have a response body.", handler.response.hasResponse());
- assertEquals("Should be correlated to the original request", request, handler.response.request());
+ assertEquals("Should be correlated to the original request", request.header(), handler.response.requestHeader());
}
private void awaitReady(NetworkClient client, Node node) {
@@ -136,11 +133,10 @@ public class NetworkClientTest {
@Test
public void testRequestTimeout() {
- ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, ByteBuffer>emptyMap());
+ ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.<TopicPartition, MemoryRecords>emptyMap());
RequestHeader reqHeader = client.nextRequestHeader(ApiKeys.PRODUCE);
- RequestSend send = new RequestSend(node.idString(), reqHeader, produceRequest.toStruct());
TestCallbackHandler handler = new TestCallbackHandler();
- ClientRequest request = new ClientRequest(time.milliseconds(), true, send, handler);
+ ClientRequest request = new ClientRequest(node.idString(), time.milliseconds(), true, reqHeader, produceRequest, handler);
awaitReady(client, node);
long now = time.milliseconds();
client.send(request, now);
@@ -211,9 +207,8 @@ public class NetworkClientTest {
awaitReady(client, node);
RequestHeader reqHeader = client.nextRequestHeader(ApiKeys.METADATA);
- Struct req = new MetadataRequest(Collections.<String>emptyList()).toStruct();
- RequestSend send = new RequestSend(node.idString(), reqHeader, req);
- ClientRequest request = new ClientRequest(time.milliseconds(), true, send, null);
+ MetadataRequest metadataRequest = new MetadataRequest(Collections.<String>emptyList());
+ ClientRequest request = new ClientRequest(node.idString(), time.milliseconds(), true, reqHeader, metadataRequest, null);
client.send(request, time.milliseconds());
client.poll(requestTimeoutMs, time.milliseconds());
assertEquals(1, client.inFlightRequestCount(node.idString()));
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index bf45ee6..fa30a4b 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -37,9 +37,9 @@ import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.network.Selectable;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.FetchResponse;
import org.apache.kafka.common.requests.FetchResponse.PartitionData;
import org.apache.kafka.common.requests.GroupCoordinatorResponse;
@@ -470,10 +470,10 @@ public class KafkaConsumerTest {
final KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor,
rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, true, autoCommitIntervalMs);
- consumer.assign(Arrays.asList(tp0));
+ consumer.assign(singletonList(tp0));
// lookup coordinator
- client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node).toStruct(), node);
+ client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node), node);
Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
// fetch offset for one topic
@@ -892,7 +892,7 @@ public class KafkaConsumerTest {
// the auto commit is disabled, so no offset commit request should be sent
for (ClientRequest req: client.requests())
- assertTrue(req.request().header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
+ assertTrue(req.header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
// subscription change
consumer.unsubscribe();
@@ -903,7 +903,7 @@ public class KafkaConsumerTest {
// the auto commit is disabled, so no offset commit request should be sent
for (ClientRequest req: client.requests())
- assertTrue(req.request().header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
+ assertTrue(req.header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
consumer.close();
}
@@ -933,7 +933,7 @@ public class KafkaConsumerTest {
rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, true, autoCommitIntervalMs);
// lookup coordinator
- client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node).toStruct(), node);
+ client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node), node);
Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
// manual assignment
@@ -997,7 +997,7 @@ public class KafkaConsumerTest {
rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, false, autoCommitIntervalMs);
// lookup coordinator
- client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node).toStruct(), node);
+ client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node), node);
Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
// manual assignment
@@ -1029,8 +1029,8 @@ public class KafkaConsumerTest {
assertTrue(consumer.assignment().equals(Collections.singleton(t2p0)));
// the auto commit is disabled, so no offset commit request should be sent
- for (ClientRequest req: client.requests())
- assertTrue(req.request().header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
+ for (ClientRequest req : client.requests())
+ assertTrue(req.header().apiKey() != ApiKeys.OFFSET_COMMIT.id);
consumer.close();
}
@@ -1086,15 +1086,15 @@ public class KafkaConsumerTest {
private Node prepareRebalance(MockClient client, Node node, final Set<String> subscribedTopics, PartitionAssignor assignor, List<TopicPartition> partitions, Node coordinator) {
if (coordinator == null) {
// lookup coordinator
- client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node).toStruct(), node);
+ client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node), node);
coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
}
// join group
client.prepareResponseFrom(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- JoinGroupRequest joinGroupRequest = new JoinGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(joinGroupRequest.groupProtocols().get(0).metadata());
return subscribedTopics.equals(new HashSet<>(subscription.topics()));
}
@@ -1109,7 +1109,7 @@ public class KafkaConsumerTest {
private Node prepareRebalance(MockClient client, Node node, PartitionAssignor assignor, List<TopicPartition> partitions, Node coordinator) {
if (coordinator == null) {
// lookup coordinator
- client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node).toStruct(), node);
+ client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE.code(), node), node);
coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
}
@@ -1126,11 +1126,11 @@ public class KafkaConsumerTest {
final AtomicBoolean heartbeatReceived = new AtomicBoolean(false);
client.prepareResponseFrom(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
heartbeatReceived.set(true);
return true;
}
- }, new HeartbeatResponse(Errors.NONE.code()).toStruct(), coordinator);
+ }, new HeartbeatResponse(Errors.NONE.code()), coordinator);
return heartbeatReceived;
}
@@ -1142,8 +1142,8 @@ public class KafkaConsumerTest {
client.prepareResponseFrom(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- OffsetCommitRequest commitRequest = new OffsetCommitRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
for (Map.Entry<TopicPartition, Long> partitionOffset : partitionOffsets.entrySet()) {
OffsetCommitRequest.PartitionData partitionData = commitRequest.offsetData().get(partitionOffset.getKey());
// verify that the expected offset has been committed
@@ -1162,39 +1162,38 @@ public class KafkaConsumerTest {
return prepareOffsetCommitResponse(client, coordinator, Collections.singletonMap(partition, offset));
}
- private Struct offsetCommitResponse(Map<TopicPartition, Short> responseData) {
- OffsetCommitResponse response = new OffsetCommitResponse(responseData);
- return response.toStruct();
+ private OffsetCommitResponse offsetCommitResponse(Map<TopicPartition, Short> responseData) {
+ return new OffsetCommitResponse(responseData);
}
- private Struct joinGroupFollowerResponse(PartitionAssignor assignor, int generationId, String memberId, String leaderId, short error) {
+ private JoinGroupResponse joinGroupFollowerResponse(PartitionAssignor assignor, int generationId, String memberId, String leaderId, short error) {
return new JoinGroupResponse(error, generationId, assignor.name(), memberId, leaderId,
- Collections.<String, ByteBuffer>emptyMap()).toStruct();
+ Collections.<String, ByteBuffer>emptyMap());
}
- private Struct syncGroupResponse(List<TopicPartition> partitions, short error) {
+ private SyncGroupResponse syncGroupResponse(List<TopicPartition> partitions, short error) {
ByteBuffer buf = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions));
- return new SyncGroupResponse(error, buf).toStruct();
+ return new SyncGroupResponse(error, buf);
}
- private Struct offsetResponse(Map<TopicPartition, Long> offsets, short error) {
+ private OffsetFetchResponse offsetResponse(Map<TopicPartition, Long> offsets, short error) {
Map<TopicPartition, OffsetFetchResponse.PartitionData> partitionData = new HashMap<>();
for (Map.Entry<TopicPartition, Long> entry : offsets.entrySet()) {
partitionData.put(entry.getKey(), new OffsetFetchResponse.PartitionData(entry.getValue(), "", error));
}
- return new OffsetFetchResponse(partitionData).toStruct();
+ return new OffsetFetchResponse(partitionData);
}
- private Struct listOffsetsResponse(Map<TopicPartition, Long> offsets, short error) {
+ private ListOffsetResponse listOffsetsResponse(Map<TopicPartition, Long> offsets, short error) {
Map<TopicPartition, ListOffsetResponse.PartitionData> partitionData = new HashMap<>();
for (Map.Entry<TopicPartition, Long> partitionOffset : offsets.entrySet()) {
partitionData.put(partitionOffset.getKey(), new ListOffsetResponse.PartitionData(error,
1L, partitionOffset.getValue()));
}
- return new ListOffsetResponse(partitionData, 1).toStruct();
+ return new ListOffsetResponse(partitionData, 1);
}
- private Struct fetchResponse(Map<TopicPartition, FetchInfo> fetches) {
+ private FetchResponse fetchResponse(Map<TopicPartition, FetchInfo> fetches) {
Map<TopicPartition, PartitionData> tpResponses = new HashMap<>();
for (Map.Entry<TopicPartition, FetchInfo> fetchEntry : fetches.entrySet()) {
TopicPartition partition = fetchEntry.getKey();
@@ -1204,13 +1203,12 @@ public class KafkaConsumerTest {
for (int i = 0; i < fetchCount; i++)
records.append(fetchOffset + i, 0L, ("key-" + i).getBytes(), ("value-" + i).getBytes());
records.close();
- tpResponses.put(partition, new FetchResponse.PartitionData(Errors.NONE.code(), 0, records.buffer()));
+ tpResponses.put(partition, new FetchResponse.PartitionData(Errors.NONE.code(), 0, records));
}
- FetchResponse response = new FetchResponse(tpResponses, 0);
- return response.toStruct();
+ return new FetchResponse(tpResponses, 0);
}
- private Struct fetchResponse(TopicPartition partition, long fetchOffset, int count) {
+ private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) {
FetchInfo fetchInfo = new FetchInfo(fetchOffset, count);
return fetchResponse(Collections.singletonMap(partition, fetchInfo));
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
index 3c8c793..bb617ae 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinatorTest.java
@@ -16,20 +16,20 @@
**/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.GroupCoordinatorResponse;
+import org.apache.kafka.common.requests.HeartbeatRequest;
import org.apache.kafka.common.requests.HeartbeatResponse;
import org.apache.kafka.common.requests.JoinGroupRequest;
import org.apache.kafka.common.requests.JoinGroupResponse;
+import org.apache.kafka.common.requests.SyncGroupRequest;
import org.apache.kafka.common.requests.SyncGroupResponse;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.common.utils.Time;
@@ -114,8 +114,8 @@ public class AbstractCoordinatorTest {
// raise the error when the background thread tries to send a heartbeat
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- if (request.request().header().apiKey() == ApiKeys.HEARTBEAT.id)
+ public boolean matches(AbstractRequest body) {
+ if (body instanceof HeartbeatRequest)
throw e;
return false;
}
@@ -160,9 +160,9 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(new MockClient.RequestMatcher() {
private int invocations = 0;
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
invocations++;
- boolean isJoinGroupRequest = request.request().header().apiKey() == ApiKeys.JOIN_GROUP.id;
+ boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
if (isJoinGroupRequest && invocations == 1)
// simulate wakeup before the request returns
throw new WakeupException();
@@ -196,9 +196,9 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(new MockClient.RequestMatcher() {
private int invocations = 0;
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
invocations++;
- boolean isJoinGroupRequest = request.request().header().apiKey() == ApiKeys.JOIN_GROUP.id;
+ boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
if (isJoinGroupRequest && invocations == 1)
// simulate wakeup before the request returns
throw new WakeupException();
@@ -233,8 +233,8 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- boolean isJoinGroupRequest = request.request().header().apiKey() == ApiKeys.JOIN_GROUP.id;
+ public boolean matches(AbstractRequest body) {
+ boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
if (isJoinGroupRequest)
// wakeup after the request returns
consumerClient.wakeup();
@@ -267,8 +267,8 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- boolean isJoinGroupRequest = request.request().header().apiKey() == ApiKeys.JOIN_GROUP.id;
+ public boolean matches(AbstractRequest body) {
+ boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
if (isJoinGroupRequest)
// wakeup after the request returns
consumerClient.wakeup();
@@ -305,9 +305,9 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(new MockClient.RequestMatcher() {
private int invocations = 0;
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
invocations++;
- boolean isSyncGroupRequest = request.request().header().apiKey() == ApiKeys.SYNC_GROUP.id;
+ boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
if (isSyncGroupRequest && invocations == 1)
// simulate wakeup after the request sent
throw new WakeupException();
@@ -341,9 +341,9 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(new MockClient.RequestMatcher() {
private int invocations = 0;
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
invocations++;
- boolean isSyncGroupRequest = request.request().header().apiKey() == ApiKeys.SYNC_GROUP.id;
+ boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
if (isSyncGroupRequest && invocations == 1)
// simulate wakeup after the request sent
throw new WakeupException();
@@ -378,8 +378,8 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- boolean isSyncGroupRequest = request.request().header().apiKey() == ApiKeys.SYNC_GROUP.id;
+ public boolean matches(AbstractRequest body) {
+ boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
if (isSyncGroupRequest)
// wakeup after the request returns
consumerClient.wakeup();
@@ -412,8 +412,8 @@ public class AbstractCoordinatorTest {
mockClient.prepareResponse(joinGroupFollowerResponse(1, "memberId", "leaderId", Errors.NONE));
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- boolean isSyncGroupRequest = request.request().header().apiKey() == ApiKeys.SYNC_GROUP.id;
+ public boolean matches(AbstractRequest body) {
+ boolean isSyncGroupRequest = body instanceof SyncGroupRequest;
if (isSyncGroupRequest)
// wakeup after the request returns
consumerClient.wakeup();
@@ -446,8 +446,8 @@ public class AbstractCoordinatorTest {
final AtomicBoolean heartbeatReceived = new AtomicBoolean(false);
mockClient.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- boolean isHeartbeatRequest = request.request().header().apiKey() == ApiKeys.HEARTBEAT.id;
+ public boolean matches(AbstractRequest body) {
+ boolean isHeartbeatRequest = body instanceof HeartbeatRequest;
if (isHeartbeatRequest)
heartbeatReceived.set(true);
return isHeartbeatRequest;
@@ -466,23 +466,21 @@ public class AbstractCoordinatorTest {
}, 3000, "Should have received a heartbeat request after joining the group");
}
- private Struct groupCoordinatorResponse(Node node, Errors error) {
- GroupCoordinatorResponse response = new GroupCoordinatorResponse(error.code(), node);
- return response.toStruct();
+ private GroupCoordinatorResponse groupCoordinatorResponse(Node node, Errors error) {
+ return new GroupCoordinatorResponse(error.code(), node);
}
- private Struct heartbeatResponse(Errors error) {
- HeartbeatResponse response = new HeartbeatResponse(error.code());
- return response.toStruct();
+ private HeartbeatResponse heartbeatResponse(Errors error) {
+ return new HeartbeatResponse(error.code());
}
- private Struct joinGroupFollowerResponse(int generationId, String memberId, String leaderId, Errors error) {
+ private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, Errors error) {
return new JoinGroupResponse(error.code(), generationId, "dummy-subprotocol", memberId, leaderId,
- Collections.<String, ByteBuffer>emptyMap()).toStruct();
+ Collections.<String, ByteBuffer>emptyMap());
}
- private Struct syncGroupResponse(Errors error) {
- return new SyncGroupResponse(error.code(), ByteBuffer.allocate(0)).toStruct();
+ private SyncGroupResponse syncGroupResponse(Errors error) {
+ return new SyncGroupResponse(error.code(), ByteBuffer.allocate(0));
}
public class DummyCoordinator extends AbstractCoordinator {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index a83d7a9..e7ba401 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -16,7 +16,6 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.consumer.CommitFailedException;
@@ -39,7 +38,7 @@ import org.apache.kafka.common.errors.OffsetMetadataTooLarge;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.GroupCoordinatorResponse;
import org.apache.kafka.common.requests.HeartbeatResponse;
import org.apache.kafka.common.requests.JoinGroupRequest;
@@ -311,8 +310,8 @@ public class ConsumerCoordinatorTest {
client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
return sync.memberId().equals(consumerId) &&
sync.generationId() == 1 &&
sync.groupAssignment().containsKey(consumerId);
@@ -353,7 +352,7 @@ public class ConsumerCoordinatorTest {
client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
final Map<String, Integer> updatedPartitions = new HashMap<>();
for (String topic : updatedSubscription)
updatedPartitions.put(topic, 1);
@@ -371,8 +370,8 @@ public class ConsumerCoordinatorTest {
// we expect to see a second rebalance with the new-found topics
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- JoinGroupRequest join = new JoinGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ JoinGroupRequest join = (JoinGroupRequest) body;
ProtocolMetadata protocolMetadata = join.groupProtocols().iterator().next();
PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata.metadata());
protocolMetadata.metadata().rewind();
@@ -443,8 +442,8 @@ public class ConsumerCoordinatorTest {
client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
return sync.memberId().equals(consumerId) &&
sync.generationId() == 1 &&
sync.groupAssignment().isEmpty();
@@ -476,13 +475,13 @@ public class ConsumerCoordinatorTest {
final AtomicBoolean received = new AtomicBoolean(false);
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
received.set(true);
- LeaveGroupRequest leaveRequest = new LeaveGroupRequest(request.request().body());
+ LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
return leaveRequest.memberId().equals(consumerId) &&
leaveRequest.groupId().equals(groupId);
}
- }, new LeaveGroupResponse(Errors.NONE.code()).toStruct());
+ }, new LeaveGroupResponse(Errors.NONE.code()));
coordinator.close();
assertTrue(received.get());
}
@@ -503,13 +502,13 @@ public class ConsumerCoordinatorTest {
final AtomicBoolean received = new AtomicBoolean(false);
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
+ public boolean matches(AbstractRequest body) {
received.set(true);
- LeaveGroupRequest leaveRequest = new LeaveGroupRequest(request.request().body());
+ LeaveGroupRequest leaveRequest = (LeaveGroupRequest) body;
return leaveRequest.memberId().equals(consumerId) &&
leaveRequest.groupId().equals(groupId);
}
- }, new LeaveGroupResponse(Errors.NONE.code()).toStruct());
+ }, new LeaveGroupResponse(Errors.NONE.code()));
coordinator.maybeLeaveGroup();
assertTrue(received.get());
@@ -548,8 +547,8 @@ public class ConsumerCoordinatorTest {
// now we should see a new join with the empty UNKNOWN_MEMBER_ID
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- JoinGroupRequest joinRequest = new JoinGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ JoinGroupRequest joinRequest = (JoinGroupRequest) body;
return joinRequest.memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
}
}, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE.code()));
@@ -600,8 +599,8 @@ public class ConsumerCoordinatorTest {
// then let the full join/sync finish successfully
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- JoinGroupRequest joinRequest = new JoinGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ JoinGroupRequest joinRequest = (JoinGroupRequest) body;
return joinRequest.memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
}
}, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE.code()));
@@ -670,8 +669,8 @@ public class ConsumerCoordinatorTest {
client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
if (sync.memberId().equals(consumerId) &&
sync.generationId() == 1 &&
sync.groupAssignment().containsKey(consumerId)) {
@@ -939,7 +938,7 @@ public class ConsumerCoordinatorTest {
coordinator.joinGroupIfNeeded();
// now switch to manual assignment
- client.prepareResponse(new LeaveGroupResponse(Errors.NONE.code()).toStruct());
+ client.prepareResponse(new LeaveGroupResponse(Errors.NONE.code()));
subscriptions.unsubscribe();
coordinator.maybeLeaveGroup();
subscriptions.assignFromUser(singleton(tp));
@@ -947,8 +946,8 @@ public class ConsumerCoordinatorTest {
// the client should not reuse generation/memberId from auto-subscribed generation
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- OffsetCommitRequest commitRequest = new OffsetCommitRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ OffsetCommitRequest commitRequest = (OffsetCommitRequest) body;
return commitRequest.memberId().equals(OffsetCommitRequest.DEFAULT_MEMBER_ID) &&
commitRequest.generationId() == OffsetCommitRequest.DEFAULT_GENERATION_ID;
}
@@ -1245,48 +1244,44 @@ public class ConsumerCoordinatorTest {
excludeInternalTopics);
}
- private Struct groupCoordinatorResponse(Node node, short error) {
- GroupCoordinatorResponse response = new GroupCoordinatorResponse(error, node);
- return response.toStruct();
+ private GroupCoordinatorResponse groupCoordinatorResponse(Node node, short error) {
+ return new GroupCoordinatorResponse(error, node);
}
- private Struct heartbeatResponse(short error) {
- HeartbeatResponse response = new HeartbeatResponse(error);
- return response.toStruct();
+ private HeartbeatResponse heartbeatResponse(short error) {
+ return new HeartbeatResponse(error);
}
- private Struct joinGroupLeaderResponse(int generationId,
- String memberId,
- Map<String, List<String>> subscriptions,
- short error) {
+ private JoinGroupResponse joinGroupLeaderResponse(int generationId,
+ String memberId,
+ Map<String, List<String>> subscriptions,
+ short error) {
Map<String, ByteBuffer> metadata = new HashMap<>();
for (Map.Entry<String, List<String>> subscriptionEntry : subscriptions.entrySet()) {
PartitionAssignor.Subscription subscription = new PartitionAssignor.Subscription(subscriptionEntry.getValue());
ByteBuffer buf = ConsumerProtocol.serializeSubscription(subscription);
metadata.put(subscriptionEntry.getKey(), buf);
}
- return new JoinGroupResponse(error, generationId, partitionAssignor.name(), memberId, memberId, metadata).toStruct();
+ return new JoinGroupResponse(error, generationId, partitionAssignor.name(), memberId, memberId, metadata);
}
- private Struct joinGroupFollowerResponse(int generationId, String memberId, String leaderId, short error) {
+ private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, short error) {
return new JoinGroupResponse(error, generationId, partitionAssignor.name(), memberId, leaderId,
- Collections.<String, ByteBuffer>emptyMap()).toStruct();
+ Collections.<String, ByteBuffer>emptyMap());
}
- private Struct syncGroupResponse(List<TopicPartition> partitions, short error) {
+ private SyncGroupResponse syncGroupResponse(List<TopicPartition> partitions, short error) {
ByteBuffer buf = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions));
- return new SyncGroupResponse(error, buf).toStruct();
+ return new SyncGroupResponse(error, buf);
}
- private Struct offsetCommitResponse(Map<TopicPartition, Short> responseData) {
- OffsetCommitResponse response = new OffsetCommitResponse(responseData);
- return response.toStruct();
+ private OffsetCommitResponse offsetCommitResponse(Map<TopicPartition, Short> responseData) {
+ return new OffsetCommitResponse(responseData);
}
- private Struct offsetFetchResponse(TopicPartition tp, Short error, String metadata, long offset) {
+ private OffsetFetchResponse offsetFetchResponse(TopicPartition tp, Short error, String metadata, long offset) {
OffsetFetchResponse.PartitionData data = new OffsetFetchResponse.PartitionData(offset, metadata, error);
- OffsetFetchResponse response = new OffsetFetchResponse(Collections.singletonMap(tp, data));
- return response.toStruct();
+ return new OffsetFetchResponse(Collections.singletonMap(tp, data));
}
private OffsetCommitCallback callback(final AtomicBoolean success) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
index f8ad3ca..003d92d 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClientTest.java
@@ -21,7 +21,6 @@ import org.apache.kafka.common.Node;
import org.apache.kafka.common.errors.WakeupException;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.requests.HeartbeatRequest;
import org.apache.kafka.common.requests.HeartbeatResponse;
import org.apache.kafka.common.utils.MockTime;
@@ -60,7 +59,7 @@ public class ConsumerNetworkClientTest {
assertTrue(future.succeeded());
ClientResponse clientResponse = future.value();
- HeartbeatResponse response = new HeartbeatResponse(clientResponse.responseBody());
+ HeartbeatResponse response = (HeartbeatResponse) clientResponse.responseBody();
assertEquals(Errors.NONE.code(), response.errorCode());
}
@@ -206,7 +205,7 @@ public class ConsumerNetworkClientTest {
client.prepareResponse(heartbeatResponse(Errors.NONE.code()));
consumerClient.poll(future2);
ClientResponse clientResponse = future2.value();
- HeartbeatResponse response = new HeartbeatResponse(clientResponse.responseBody());
+ HeartbeatResponse response = (HeartbeatResponse) clientResponse.responseBody();
assertEquals(Errors.NONE.code(), response.errorCode());
// Disable ready flag to delay send and queue another send. Disconnection should remove pending send
@@ -226,9 +225,8 @@ public class ConsumerNetworkClientTest {
return new HeartbeatRequest("group", 1, "memberId");
}
- private Struct heartbeatResponse(short error) {
- HeartbeatResponse response = new HeartbeatResponse(error);
- return response.toStruct();
+ private HeartbeatResponse heartbeatResponse(short error) {
+ return new HeartbeatResponse(error);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 5822646..140e041 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -16,12 +16,12 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.clients.consumer.OffsetOutOfRangeException;
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
import org.apache.kafka.common.Cluster;
@@ -37,12 +37,11 @@ import org.apache.kafka.common.errors.TopicAuthorizationException;
import org.apache.kafka.common.metrics.KafkaMetric;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.record.Compressor;
import org.apache.kafka.common.record.MemoryRecords;
-import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.common.record.Record;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.FetchResponse;
import org.apache.kafka.common.requests.ListOffsetRequest;
@@ -166,8 +165,8 @@ public class FetcherTest {
private MockClient.RequestMatcher matchesOffset(final TopicPartition tp, final long offset) {
return new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- FetchRequest fetch = new FetchRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ FetchRequest fetch = (FetchRequest) body;
return fetch.fetchData().containsKey(tp) &&
fetch.fetchData().get(tp).offset == offset;
}
@@ -542,7 +541,7 @@ public class FetcherTest {
@Test
public void testGetAllTopics() {
// sending response before request, as getTopicMetadata is a blocking call
- client.prepareResponse(newMetadataResponse(topicName, Errors.NONE).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.NONE));
Map<String, List<PartitionInfo>> allTopics = fetcher.getAllTopicMetadata(5000L);
@@ -553,7 +552,7 @@ public class FetcherTest {
public void testGetAllTopicsDisconnect() {
// first try gets a disconnect, next succeeds
client.prepareResponse(null, true);
- client.prepareResponse(newMetadataResponse(topicName, Errors.NONE).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.NONE));
Map<String, List<PartitionInfo>> allTopics = fetcher.getAllTopicMetadata(5000L);
assertEquals(cluster.topics().size(), allTopics.size());
}
@@ -566,7 +565,7 @@ public class FetcherTest {
@Test
public void testGetAllTopicsUnauthorized() {
- client.prepareResponse(newMetadataResponse(topicName, Errors.TOPIC_AUTHORIZATION_FAILED).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.TOPIC_AUTHORIZATION_FAILED));
try {
fetcher.getAllTopicMetadata(10L);
fail();
@@ -577,13 +576,13 @@ public class FetcherTest {
@Test(expected = InvalidTopicException.class)
public void testGetTopicMetadataInvalidTopic() {
- client.prepareResponse(newMetadataResponse(topicName, Errors.INVALID_TOPIC_EXCEPTION).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.INVALID_TOPIC_EXCEPTION));
fetcher.getTopicMetadata(new MetadataRequest(Collections.singletonList(topicName)), 5000L);
}
@Test
public void testGetTopicMetadataUnknownTopic() {
- client.prepareResponse(newMetadataResponse(topicName, Errors.UNKNOWN_TOPIC_OR_PARTITION).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.UNKNOWN_TOPIC_OR_PARTITION));
Map<String, List<PartitionInfo>> topicMetadata = fetcher.getTopicMetadata(new MetadataRequest(Collections.singletonList(topicName)), 5000L);
assertNull(topicMetadata.get(topicName));
@@ -591,8 +590,8 @@ public class FetcherTest {
@Test
public void testGetTopicMetadataLeaderNotAvailable() {
- client.prepareResponse(newMetadataResponse(topicName, Errors.LEADER_NOT_AVAILABLE).toStruct());
- client.prepareResponse(newMetadataResponse(topicName, Errors.NONE).toStruct());
+ client.prepareResponse(newMetadataResponse(topicName, Errors.LEADER_NOT_AVAILABLE));
+ client.prepareResponse(newMetadataResponse(topicName, Errors.NONE));
Map<String, List<PartitionInfo>> topicMetadata = fetcher.getTopicMetadata(new MetadataRequest(Collections.singletonList(topicName)), 5000L);
assertTrue(topicMetadata.containsKey(topicName));
@@ -705,28 +704,27 @@ public class FetcherTest {
// matches any list offset request with the provided timestamp
return new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- ListOffsetRequest req = new ListOffsetRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ ListOffsetRequest req = (ListOffsetRequest) body;
return timestamp == req.partitionTimestamps().get(tp);
}
};
}
- private Struct listOffsetResponse(Errors error, long timestamp, long offset) {
+ private ListOffsetResponse listOffsetResponse(Errors error, long timestamp, long offset) {
return listOffsetResponse(tp, error, timestamp, offset);
}
- private Struct listOffsetResponse(TopicPartition tp, Errors error, long timestamp, long offset) {
+ private ListOffsetResponse listOffsetResponse(TopicPartition tp, Errors error, long timestamp, long offset) {
ListOffsetResponse.PartitionData partitionData = new ListOffsetResponse.PartitionData(error.code(), timestamp, offset);
Map<TopicPartition, ListOffsetResponse.PartitionData> allPartitionData = new HashMap<>();
allPartitionData.put(tp, partitionData);
- ListOffsetResponse response = new ListOffsetResponse(allPartitionData, 1);
- return response.toStruct();
+ return new ListOffsetResponse(allPartitionData, 1);
}
- private Struct fetchResponse(ByteBuffer buffer, short error, long hw, int throttleTime) {
- FetchResponse response = new FetchResponse(Collections.singletonMap(tp, new FetchResponse.PartitionData(error, hw, buffer)), throttleTime);
- return response.toStruct();
+ private FetchResponse fetchResponse(ByteBuffer buffer, short error, long hw, int throttleTime) {
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ return new FetchResponse(Collections.singletonMap(tp, new FetchResponse.PartitionData(error, hw, records)), throttleTime);
}
private MetadataResponse newMetadataResponse(String topic, Errors error) {
[5/5] kafka git commit: KAFKA-2066;
Use client-side FetchRequest/FetchResponse on server
Posted by jg...@apache.org.
KAFKA-2066; Use client-side FetchRequest/FetchResponse on server
Author: Jason Gustafson <ja...@confluent.io>
Reviewers: Jun Rao <ju...@gmail.com>
Closes #2069 from hachikuji/KAFKA-2066
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/3b4c3479
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/3b4c3479
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/3b4c3479
Branch: refs/heads/trunk
Commit: 3b4c347949c02b1e2b1dd473deda0f8d2304d027
Parents: 1f1d450
Author: Jason Gustafson <ja...@confluent.io>
Authored: Mon Nov 14 16:31:04 2016 -0800
Committer: Jason Gustafson <ja...@confluent.io>
Committed: Mon Nov 14 16:31:04 2016 -0800
----------------------------------------------------------------------
checkstyle/import-control.xml | 2 +
.../org/apache/kafka/clients/ClientRequest.java | 65 +++---
.../apache/kafka/clients/ClientResponse.java | 56 +++--
.../apache/kafka/clients/InFlightRequests.java | 46 ++--
.../org/apache/kafka/clients/KafkaClient.java | 1 -
.../kafka/clients/ManualMetadataUpdater.java | 11 +-
.../apache/kafka/clients/MetadataUpdater.java | 10 +-
.../org/apache/kafka/clients/NetworkClient.java | 164 ++++++++------
.../consumer/internals/AbstractCoordinator.java | 33 +--
.../consumer/internals/ConsumerCoordinator.java | 12 -
.../internals/ConsumerNetworkClient.java | 30 ++-
.../clients/consumer/internals/Fetcher.java | 11 +-
.../clients/producer/internals/Sender.java | 68 +++---
.../kafka/common/network/ByteBufferSend.java | 5 +-
.../kafka/common/network/NetworkSend.java | 4 +-
.../apache/kafka/common/protocol/ApiKeys.java | 2 +-
.../apache/kafka/common/protocol/Protocol.java | 21 +-
.../kafka/common/protocol/types/ArrayOf.java | 13 +-
.../kafka/common/protocol/types/Schema.java | 18 +-
.../kafka/common/protocol/types/Struct.java | 16 +-
.../kafka/common/protocol/types/Type.java | 48 ++++
.../apache/kafka/common/record/FileRecords.java | 126 +++++++++++
.../kafka/common/record/LogInputStream.java | 35 +++
.../kafka/common/record/MemoryRecords.java | 186 ++++------------
.../org/apache/kafka/common/record/Records.java | 15 ++
.../kafka/common/record/RecordsIterator.java | 170 ++++++++++++++
.../kafka/common/requests/AbstractRequest.java | 8 +-
.../requests/AbstractRequestResponse.java | 9 +-
.../kafka/common/requests/AbstractResponse.java | 86 ++++++++
.../common/requests/ApiVersionsRequest.java | 2 +-
.../common/requests/ApiVersionsResponse.java | 2 +-
.../requests/ControlledShutdownRequest.java | 2 +-
.../requests/ControlledShutdownResponse.java | 2 +-
.../common/requests/CreateTopicsRequest.java | 2 +-
.../common/requests/CreateTopicsResponse.java | 2 +-
.../common/requests/DeleteTopicsRequest.java | 2 +-
.../common/requests/DeleteTopicsResponse.java | 2 +-
.../common/requests/DescribeGroupsRequest.java | 2 +-
.../common/requests/DescribeGroupsResponse.java | 2 +-
.../kafka/common/requests/FetchRequest.java | 17 +-
.../kafka/common/requests/FetchResponse.java | 163 ++++++++++----
.../requests/GroupCoordinatorRequest.java | 2 +-
.../requests/GroupCoordinatorResponse.java | 2 +-
.../kafka/common/requests/HeartbeatRequest.java | 2 +-
.../common/requests/HeartbeatResponse.java | 2 +-
.../kafka/common/requests/JoinGroupRequest.java | 2 +-
.../common/requests/JoinGroupResponse.java | 2 +-
.../common/requests/LeaderAndIsrRequest.java | 2 +-
.../common/requests/LeaderAndIsrResponse.java | 2 +-
.../common/requests/LeaveGroupRequest.java | 2 +-
.../common/requests/LeaveGroupResponse.java | 2 +-
.../common/requests/ListGroupsRequest.java | 2 +-
.../common/requests/ListGroupsResponse.java | 2 +-
.../common/requests/ListOffsetRequest.java | 2 +-
.../common/requests/ListOffsetResponse.java | 2 +-
.../kafka/common/requests/MetadataRequest.java | 2 +-
.../kafka/common/requests/MetadataResponse.java | 2 +-
.../common/requests/OffsetCommitRequest.java | 8 +-
.../common/requests/OffsetCommitResponse.java | 2 +-
.../common/requests/OffsetFetchRequest.java | 10 +-
.../common/requests/OffsetFetchResponse.java | 2 +-
.../kafka/common/requests/ProduceRequest.java | 31 +--
.../kafka/common/requests/ProduceResponse.java | 8 +-
.../kafka/common/requests/RecordsSend.java | 77 +++++++
.../kafka/common/requests/RequestSend.java | 55 -----
.../kafka/common/requests/ResponseSend.java | 41 ----
.../common/requests/SaslHandshakeRequest.java | 2 +-
.../common/requests/SaslHandshakeResponse.java | 2 +-
.../common/requests/StopReplicaRequest.java | 2 +-
.../common/requests/StopReplicaResponse.java | 2 +-
.../kafka/common/requests/SyncGroupRequest.java | 2 +-
.../common/requests/SyncGroupResponse.java | 2 +-
.../common/requests/UpdateMetadataRequest.java | 2 +-
.../common/requests/UpdateMetadataResponse.java | 2 +-
.../authenticator/SaslClientAuthenticator.java | 57 +++--
.../authenticator/SaslServerAuthenticator.java | 71 +++---
.../kafka/common/utils/AbstractIterator.java | 4 +-
.../org/apache/kafka/clients/MockClient.java | 101 ++++-----
.../apache/kafka/clients/NetworkClientTest.java | 31 ++-
.../clients/consumer/KafkaConsumerTest.java | 64 +++---
.../internals/AbstractCoordinatorTest.java | 64 +++---
.../internals/ConsumerCoordinatorTest.java | 85 ++++---
.../internals/ConsumerNetworkClientTest.java | 10 +-
.../clients/consumer/internals/FetcherTest.java | 40 ++--
.../clients/producer/internals/SenderTest.java | 40 ++--
.../common/requests/RequestResponseTest.java | 174 +++++++++++----
.../authenticator/SaslAuthenticatorTest.java | 54 ++---
.../distributed/WorkerCoordinatorTest.java | 41 ++--
.../main/scala/kafka/admin/AdminClient.scala | 18 +-
.../src/main/scala/kafka/api/FetchRequest.scala | 3 +-
.../main/scala/kafka/api/FetchResponse.scala | 3 +-
.../kafka/api/GenericRequestAndHeader.scala | 5 +-
.../kafka/api/GenericResponseAndHeader.scala | 5 +-
.../main/scala/kafka/api/ProducerRequest.scala | 4 +-
.../scala/kafka/api/RequestOrResponse.scala | 2 +-
.../controller/ControllerChannelManager.scala | 59 +++--
.../kafka/controller/KafkaController.scala | 8 +-
.../kafka/controller/TopicDeletionManager.scala | 7 +-
.../main/scala/kafka/log/FileMessageSet.scala | 35 +--
.../kafka/message/ByteBufferMessageSet.scala | 17 +-
.../main/scala/kafka/message/MessageSet.scala | 12 +-
.../scala/kafka/network/RequestChannel.scala | 45 ++--
.../main/scala/kafka/server/DelayedFetch.scala | 9 +-
.../src/main/scala/kafka/server/KafkaApis.scala | 219 +++++++++----------
.../main/scala/kafka/server/KafkaServer.scala | 58 +++--
.../kafka/server/ReplicaFetcherThread.scala | 24 +-
.../scala/kafka/server/ReplicaManager.scala | 31 +--
.../kafka/server/ReplicationQuotaManager.scala | 2 +-
.../kafka/utils/NetworkClientBlockingOps.scala | 14 +-
.../kafka/api/AuthorizerIntegrationTest.scala | 20 +-
.../kafka/message/BaseMessageSetTestCases.scala | 4 +-
.../unit/kafka/network/SocketServerTest.scala | 7 +-
.../kafka/server/CreateTopicsRequestTest.scala | 2 +-
.../unit/kafka/server/EdgeCaseRequestTest.scala | 7 +-
.../unit/kafka/server/FetchRequestTest.scala | 10 +-
.../unit/kafka/server/LeaderElectionTest.scala | 4 +-
.../unit/kafka/server/ProduceRequestTest.scala | 6 +-
.../kafka/server/ReplicaManagerQuotasTest.scala | 11 +-
.../unit/kafka/server/ReplicaManagerTest.scala | 9 +-
.../unit/kafka/server/SimpleFetchTest.scala | 15 +-
120 files changed, 1870 insertions(+), 1394 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/checkstyle/import-control.xml
----------------------------------------------------------------------
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 7716f43..58525ad 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -81,11 +81,13 @@
<subpackage name="protocol">
<allow pkg="org.apache.kafka.common.errors" />
<allow pkg="org.apache.kafka.common.protocol.types" />
+ <allow pkg="org.apache.kafka.common.record" />
</subpackage>
<subpackage name="record">
<allow pkg="net.jpountz" />
<allow pkg="org.apache.kafka.common.record" />
+ <allow pkg="org.apache.kafka.common.network" />
</subpackage>
<subpackage name="requests">
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
index 117b0bf..de6e506 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java
@@ -12,56 +12,50 @@
*/
package org.apache.kafka.clients;
-import org.apache.kafka.common.requests.RequestSend;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.RequestHeader;
/**
* A request being sent to the server. This holds both the network send as well as the client-level metadata.
*/
public final class ClientRequest {
+ private final String destination;
+ private final RequestHeader header;
+ private final AbstractRequest body;
private final long createdTimeMs;
private final boolean expectResponse;
- private final RequestSend request;
private final RequestCompletionHandler callback;
- private final boolean isInitiatedByNetworkClient;
- private long sendTimeMs;
/**
+ * @param destination The brokerId to send the request to
* @param createdTimeMs The unix timestamp in milliseconds for the time at which this request was created.
* @param expectResponse Should we expect a response message or is this request complete once it is sent?
- * @param request The request
+ * @param header The request's header
+ * @param body The request's body
* @param callback A callback to execute when the response has been received (or null if no callback is necessary)
*/
- public ClientRequest(long createdTimeMs, boolean expectResponse, RequestSend request,
+ public ClientRequest(String destination,
+ long createdTimeMs,
+ boolean expectResponse,
+ RequestHeader header,
+ AbstractRequest body,
RequestCompletionHandler callback) {
- this(createdTimeMs, expectResponse, request, callback, false);
- }
-
- /**
- * @param createdTimeMs The unix timestamp in milliseconds for the time at which this request was created.
- * @param expectResponse Should we expect a response message or is this request complete once it is sent?
- * @param request The request
- * @param callback A callback to execute when the response has been received (or null if no callback is necessary)
- * @param isInitiatedByNetworkClient Is request initiated by network client, if yes, its
- * response will be consumed by network client
- */
- public ClientRequest(long createdTimeMs, boolean expectResponse, RequestSend request,
- RequestCompletionHandler callback, boolean isInitiatedByNetworkClient) {
+ this.destination = destination;
this.createdTimeMs = createdTimeMs;
this.callback = callback;
- this.request = request;
+ this.header = header;
+ this.body = body;
this.expectResponse = expectResponse;
- this.isInitiatedByNetworkClient = isInitiatedByNetworkClient;
}
@Override
public String toString() {
return "ClientRequest(expectResponse=" + expectResponse +
", callback=" + callback +
- ", request=" + request +
- (isInitiatedByNetworkClient ? ", isInitiatedByNetworkClient" : "") +
+ ", header=" + header +
+ ", body=" + body +
", createdTimeMs=" + createdTimeMs +
- ", sendTimeMs=" + sendTimeMs +
")";
}
@@ -69,12 +63,16 @@ public final class ClientRequest {
return expectResponse;
}
- public RequestSend request() {
- return request;
+ public RequestHeader header() {
+ return header;
}
- public boolean hasCallback() {
- return callback != null;
+ public AbstractRequest body() {
+ return body;
+ }
+
+ public String destination() {
+ return destination;
}
public RequestCompletionHandler callback() {
@@ -85,15 +83,4 @@ public final class ClientRequest {
return createdTimeMs;
}
- public boolean isInitiatedByNetworkClient() {
- return isInitiatedByNetworkClient;
- }
-
- public long sendTimeMs() {
- return sendTimeMs;
- }
-
- public void setSendTimeMs(long sendTimeMs) {
- this.sendTimeMs = sendTimeMs;
- }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/ClientResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientResponse.java b/clients/src/main/java/org/apache/kafka/clients/ClientResponse.java
index 3b6f955..3cd8f1a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientResponse.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientResponse.java
@@ -12,30 +12,45 @@
*/
package org.apache.kafka.clients;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.RequestHeader;
/**
- * A response from the server. Contains both the body of the response as well as the correlated request that was
- * originally sent.
+ * A response from the server. Contains both the body of the response as well as the correlated request
+ * metadata that was originally sent.
*/
public class ClientResponse {
+ private final RequestHeader requestHeader;
+ private final RequestCompletionHandler callback;
+ private final String destination;
private final long receivedTimeMs;
+ private final long latencyMs;
private final boolean disconnected;
- private final ClientRequest request;
- private final Struct responseBody;
+ private final AbstractResponse responseBody;
/**
- * @param request The original request
+ * @param requestHeader The header of the corresponding request
+ * @param callback The callback to be invoked
+ * @param createdTimeMs The unix timestamp when the corresponding request was created
+ * @param destination The node the corresponding request was sent to
* @param receivedTimeMs The unix timestamp when this response was received
* @param disconnected Whether the client disconnected before fully reading a response
* @param responseBody The response contents (or null) if we disconnected or no response was expected
*/
- public ClientResponse(ClientRequest request, long receivedTimeMs, boolean disconnected, Struct responseBody) {
- super();
+ public ClientResponse(RequestHeader requestHeader,
+ RequestCompletionHandler callback,
+ String destination,
+ long createdTimeMs,
+ long receivedTimeMs,
+ boolean disconnected,
+ AbstractResponse responseBody) {
+ this.requestHeader = requestHeader;
+ this.callback = callback;
+ this.destination = destination;
this.receivedTimeMs = receivedTimeMs;
+ this.latencyMs = receivedTimeMs - createdTimeMs;
this.disconnected = disconnected;
- this.request = request;
this.responseBody = responseBody;
}
@@ -47,11 +62,15 @@ public class ClientResponse {
return disconnected;
}
- public ClientRequest request() {
- return request;
+ public RequestHeader requestHeader() {
+ return requestHeader;
}
- public Struct responseBody() {
+ public String destination() {
+ return destination;
+ }
+
+ public AbstractResponse responseBody() {
return responseBody;
}
@@ -60,16 +79,23 @@ public class ClientResponse {
}
public long requestLatencyMs() {
- return receivedTimeMs() - this.request.createdTimeMs();
+ return latencyMs;
+ }
+
+ public void onComplete() {
+ if (callback != null)
+ callback.onComplete(this);
}
@Override
public String toString() {
return "ClientResponse(receivedTimeMs=" + receivedTimeMs +
+ ", latencyMs=" +
+ latencyMs +
", disconnected=" +
disconnected +
- ", request=" +
- request +
+ ", requestHeader=" +
+ requestHeader +
", responseBody=" +
responseBody +
")";
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
index 91b9dba..f4f753e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
+++ b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java
@@ -26,7 +26,7 @@ import java.util.Map;
final class InFlightRequests {
private final int maxInFlightRequestsPerConnection;
- private final Map<String, Deque<ClientRequest>> requests = new HashMap<>();
+ private final Map<String, Deque<NetworkClient.InFlightRequest>> requests = new HashMap<>();
public InFlightRequests(int maxInFlightRequestsPerConnection) {
this.maxInFlightRequestsPerConnection = maxInFlightRequestsPerConnection;
@@ -35,11 +35,12 @@ final class InFlightRequests {
/**
* Add the given request to the queue for the connection it was directed to
*/
- public void add(ClientRequest request) {
- Deque<ClientRequest> reqs = this.requests.get(request.request().destination());
+ public void add(NetworkClient.InFlightRequest request) {
+ String destination = request.destination;
+ Deque<NetworkClient.InFlightRequest> reqs = this.requests.get(destination);
if (reqs == null) {
reqs = new ArrayDeque<>();
- this.requests.put(request.request().destination(), reqs);
+ this.requests.put(destination, reqs);
}
reqs.addFirst(request);
}
@@ -47,8 +48,8 @@ final class InFlightRequests {
/**
* Get the request queue for the given node
*/
- private Deque<ClientRequest> requestQueue(String node) {
- Deque<ClientRequest> reqs = requests.get(node);
+ private Deque<NetworkClient.InFlightRequest> requestQueue(String node) {
+ Deque<NetworkClient.InFlightRequest> reqs = requests.get(node);
if (reqs == null || reqs.isEmpty())
throw new IllegalStateException("Response from server for which there are no in-flight requests.");
return reqs;
@@ -57,7 +58,7 @@ final class InFlightRequests {
/**
* Get the oldest request (the one that that will be completed next) for the given node
*/
- public ClientRequest completeNext(String node) {
+ public NetworkClient.InFlightRequest completeNext(String node) {
return requestQueue(node).pollLast();
}
@@ -65,7 +66,7 @@ final class InFlightRequests {
* Get the last request we sent to the given node (but don't remove it from the queue)
* @param node The node id
*/
- public ClientRequest lastSent(String node) {
+ public NetworkClient.InFlightRequest lastSent(String node) {
return requestQueue(node).peekFirst();
}
@@ -74,20 +75,20 @@ final class InFlightRequests {
* @param node The node the request was sent to
* @return The request
*/
- public ClientRequest completeLastSent(String node) {
+ public NetworkClient.InFlightRequest completeLastSent(String node) {
return requestQueue(node).pollFirst();
}
/**
* Can we send more requests to this node?
- *
+ *
* @param node Node in question
* @return true iff we have no requests still being sent to the given node
*/
public boolean canSendMore(String node) {
- Deque<ClientRequest> queue = requests.get(node);
+ Deque<NetworkClient.InFlightRequest> queue = requests.get(node);
return queue == null || queue.isEmpty() ||
- (queue.peekFirst().request().completed() && queue.size() < this.maxInFlightRequestsPerConnection);
+ (queue.peekFirst().send.completed() && queue.size() < this.maxInFlightRequestsPerConnection);
}
/**
@@ -96,7 +97,7 @@ final class InFlightRequests {
* @return The request count.
*/
public int inFlightRequestCount(String node) {
- Deque<ClientRequest> queue = requests.get(node);
+ Deque<NetworkClient.InFlightRequest> queue = requests.get(node);
return queue == null ? 0 : queue.size();
}
@@ -105,19 +106,19 @@ final class InFlightRequests {
*/
public int inFlightRequestCount() {
int total = 0;
- for (Deque<ClientRequest> deque : this.requests.values())
+ for (Deque<NetworkClient.InFlightRequest> deque : this.requests.values())
total += deque.size();
return total;
}
/**
* Clear out all the in-flight requests for the given node and return them
- *
+ *
* @param node The node
* @return All the in-flight requests for that node that have been removed
*/
- public Iterable<ClientRequest> clearAll(String node) {
- Deque<ClientRequest> reqs = requests.get(node);
+ public Iterable<NetworkClient.InFlightRequest> clearAll(String node) {
+ Deque<NetworkClient.InFlightRequest> reqs = requests.get(node);
if (reqs == null) {
return Collections.emptyList();
} else {
@@ -126,7 +127,7 @@ final class InFlightRequests {
}
/**
- * Returns a list of nodes with pending inflight request, that need to be timed out
+ * Returns a list of nodes with pending in-flight request, that need to be timed out
*
* @param now current time in milliseconds
* @param requestTimeout max time to wait for the request to be completed
@@ -134,13 +135,13 @@ final class InFlightRequests {
*/
public List<String> getNodesWithTimedOutRequests(long now, int requestTimeout) {
List<String> nodeIds = new LinkedList<>();
- for (Map.Entry<String, Deque<ClientRequest>> requestEntry : requests.entrySet()) {
+ for (Map.Entry<String, Deque<NetworkClient.InFlightRequest>> requestEntry : requests.entrySet()) {
String nodeId = requestEntry.getKey();
- Deque<ClientRequest> deque = requestEntry.getValue();
+ Deque<NetworkClient.InFlightRequest> deque = requestEntry.getValue();
if (!deque.isEmpty()) {
- ClientRequest request = deque.peekLast();
- long timeSinceSend = now - request.sendTimeMs();
+ NetworkClient.InFlightRequest request = deque.peekLast();
+ long timeSinceSend = now - request.sendTimeMs;
if (timeSinceSend > requestTimeout)
nodeIds.add(nodeId);
}
@@ -148,4 +149,5 @@ final class InFlightRequests {
return nodeIds;
}
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java b/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java
index 8c6e39a..f171d13 100644
--- a/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java
@@ -66,7 +66,6 @@ public interface KafkaClient extends Closeable {
/**
* Queue up the given request for sending. Requests can only be sent on ready connections.
- *
* @param request The request
* @param now The current timestamp
*/
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/ManualMetadataUpdater.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/ManualMetadataUpdater.java b/clients/src/main/java/org/apache/kafka/clients/ManualMetadataUpdater.java
index efbe664..1c9fa79 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ManualMetadataUpdater.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ManualMetadataUpdater.java
@@ -14,7 +14,8 @@
package org.apache.kafka.clients;
import org.apache.kafka.common.Node;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.RequestHeader;
import java.util.ArrayList;
import java.util.List;
@@ -60,13 +61,13 @@ public class ManualMetadataUpdater implements MetadataUpdater {
}
@Override
- public boolean maybeHandleDisconnection(ClientRequest request) {
- return false;
+ public void handleDisconnection(String destination) {
+ // Do nothing
}
@Override
- public boolean maybeHandleCompletedReceive(ClientRequest request, long now, Struct body) {
- return false;
+ public void handleCompletedMetadataResponse(RequestHeader requestHeader, long now, AbstractResponse body) {
+ // Do nothing
}
@Override
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java b/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java
index 4669a68..21c50bd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java
+++ b/clients/src/main/java/org/apache/kafka/clients/MetadataUpdater.java
@@ -14,7 +14,8 @@
package org.apache.kafka.clients;
import org.apache.kafka.common.Node;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.RequestHeader;
import java.util.List;
@@ -41,7 +42,7 @@ interface MetadataUpdater {
* be 0 if an update has been started as a result of this call).
*
* If the implementation relies on `NetworkClient` to send requests, the completed receive will be passed to
- * `maybeHandleCompletedReceive`.
+ * `handleCompletedMetadataResponse`.
*
* The semantics of `needed` and `possible` are implementation-dependent and may take into account a number of
* factors like node availability, how long since the last metadata update, etc.
@@ -53,8 +54,9 @@ interface MetadataUpdater {
*
* This provides a mechanism for the `MetadataUpdater` implementation to use the NetworkClient instance for its own
* requests with special handling for disconnections of such requests.
+ * @param destination
*/
- boolean maybeHandleDisconnection(ClientRequest request);
+ void handleDisconnection(String destination);
/**
* If `request` is a metadata request, handles it and returns `true`. Otherwise, returns `false`.
@@ -62,7 +64,7 @@ interface MetadataUpdater {
* This provides a mechanism for the `MetadataUpdater` implementation to use the NetworkClient instance for its own
* requests with special handling for completed receives of such requests.
*/
- boolean maybeHandleCompletedReceive(ClientRequest request, long now, Struct body);
+ void handleCompletedMetadataResponse(RequestHeader requestHeader, long now, AbstractResponse body);
/**
* Schedules an update of the current cluster metadata info. A subsequent call to `maybeUpdate` would trigger the
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
index 29c6d6f..124810d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java
@@ -21,10 +21,10 @@ import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.RequestSend;
import org.apache.kafka.common.requests.ResponseHeader;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
@@ -173,8 +173,9 @@ public class NetworkClient implements KafkaClient {
@Override
public void close(String nodeId) {
selector.close(nodeId);
- for (ClientRequest request : inFlightRequests.clearAll(nodeId))
- metadataUpdater.maybeHandleDisconnection(request);
+ for (InFlightRequest request : inFlightRequests.clearAll(nodeId))
+ if (request.isInternalMetadataRequest)
+ metadataUpdater.handleDisconnection(request.destination);
connectionStates.remove(nodeId);
}
@@ -230,22 +231,38 @@ public class NetworkClient implements KafkaClient {
/**
* Queue up the given request for sending. Requests can only be sent out to ready nodes.
- *
* @param request The request
* @param now The current timestamp
*/
@Override
public void send(ClientRequest request, long now) {
- String nodeId = request.request().destination();
+ doSend(request, false, now);
+ }
+
+ private void sendInternalMetadataRequest(MetadataRequest metadataRequest, String nodeConnectionId, long now) {
+ ClientRequest clientRequest = new ClientRequest(nodeConnectionId, now, true,
+ nextRequestHeader(ApiKeys.METADATA), metadataRequest, null);
+ doSend(clientRequest, true, now);
+ }
+
+ private void doSend(ClientRequest request, boolean isInternalMetadataRequest, long now) {
+ String nodeId = request.destination();
if (!canSendRequest(nodeId))
throw new IllegalStateException("Attempt to send a request to node " + nodeId + " which is not ready.");
- doSend(request, now);
- }
- private void doSend(ClientRequest request, long now) {
- request.setSendTimeMs(now);
- this.inFlightRequests.add(request);
- selector.send(request.request());
+ Send send = request.body().toSend(nodeId, request.header());
+ InFlightRequest inFlightRequest = new InFlightRequest(
+ request.header(),
+ request.createdTimeMs(),
+ request.destination(),
+ request.callback(),
+ request.expectResponse(),
+ isInternalMetadataRequest,
+ send,
+ now);
+
+ this.inFlightRequests.add(inFlightRequest);
+ selector.send(inFlightRequest.send);
}
/**
@@ -277,12 +294,10 @@ public class NetworkClient implements KafkaClient {
// invoke callbacks
for (ClientResponse response : responses) {
- if (response.request().hasCallback()) {
- try {
- response.request().callback().onComplete(response);
- } catch (Exception e) {
- log.error("Uncaught error in request completion:", e);
- }
+ try {
+ response.onComplete();
+ } catch (Exception e) {
+ log.error("Uncaught error in request completion:", e);
}
}
@@ -376,14 +391,14 @@ public class NetworkClient implements KafkaClient {
return found;
}
- public static Struct parseResponse(ByteBuffer responseBuffer, RequestHeader requestHeader) {
+ public static AbstractResponse parseResponse(ByteBuffer responseBuffer, RequestHeader requestHeader) {
ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer);
// Always expect the response version id to be the same as the request version id
short apiKey = requestHeader.apiKey();
short apiVer = requestHeader.apiVersion();
Struct responseBody = ProtoUtils.responseSchema(apiKey, apiVer).read(responseBuffer);
correlate(requestHeader, responseHeader);
- return responseBody;
+ return AbstractResponse.getResponse(apiKey, responseBody);
}
/**
@@ -395,10 +410,12 @@ public class NetworkClient implements KafkaClient {
*/
private void processDisconnection(List<ClientResponse> responses, String nodeId, long now) {
connectionStates.disconnected(nodeId, now);
- for (ClientRequest request : this.inFlightRequests.clearAll(nodeId)) {
+ for (InFlightRequest request : this.inFlightRequests.clearAll(nodeId)) {
log.trace("Cancelled request {} due to node {} being disconnected", request, nodeId);
- if (!metadataUpdater.maybeHandleDisconnection(request))
- responses.add(new ClientResponse(request, now, true, null));
+ if (request.isInternalMetadataRequest)
+ metadataUpdater.handleDisconnection(request.destination);
+ else
+ responses.add(request.disconnected(now));
}
}
@@ -432,10 +449,10 @@ public class NetworkClient implements KafkaClient {
private void handleCompletedSends(List<ClientResponse> responses, long now) {
// if no response is expected then when the send is completed, return it
for (Send send : this.selector.completedSends()) {
- ClientRequest request = this.inFlightRequests.lastSent(send.destination());
- if (!request.expectResponse()) {
+ InFlightRequest request = this.inFlightRequests.lastSent(send.destination());
+ if (!request.expectResponse) {
this.inFlightRequests.completeLastSent(send.destination());
- responses.add(new ClientResponse(request, now, false, null));
+ responses.add(request.completed(null, now));
}
}
}
@@ -449,10 +466,12 @@ public class NetworkClient implements KafkaClient {
private void handleCompletedReceives(List<ClientResponse> responses, long now) {
for (NetworkReceive receive : this.selector.completedReceives()) {
String source = receive.source();
- ClientRequest req = inFlightRequests.completeNext(source);
- Struct body = parseResponse(receive.payload(), req.request().header());
- if (!metadataUpdater.maybeHandleCompletedReceive(req, now, body))
- responses.add(new ClientResponse(req, now, false, body));
+ InFlightRequest req = inFlightRequests.completeNext(source);
+ AbstractResponse body = parseResponse(receive.payload(), req.header);
+ if (req.isInternalMetadataRequest)
+ metadataUpdater.handleCompletedMetadataResponse(req.header, now, body);
+ else
+ responses.add(req.completed(body, now));
}
}
@@ -558,33 +577,23 @@ public class NetworkClient implements KafkaClient {
}
@Override
- public boolean maybeHandleDisconnection(ClientRequest request) {
- ApiKeys requestKey = ApiKeys.forId(request.request().header().apiKey());
-
- if (requestKey == ApiKeys.METADATA && request.isInitiatedByNetworkClient()) {
- Cluster cluster = metadata.fetch();
- if (cluster.isBootstrapConfigured()) {
- int nodeId = Integer.parseInt(request.request().destination());
- Node node = cluster.nodeById(nodeId);
- if (node != null)
- log.warn("Bootstrap broker {}:{} disconnected", node.host(), node.port());
- }
-
- metadataFetchInProgress = false;
- return true;
+ public void handleDisconnection(String destination) {
+ Cluster cluster = metadata.fetch();
+ if (cluster.isBootstrapConfigured()) {
+ int nodeId = Integer.parseInt(destination);
+ Node node = cluster.nodeById(nodeId);
+ if (node != null)
+ log.warn("Bootstrap broker {}:{} disconnected", node.host(), node.port());
}
- return false;
+ metadataFetchInProgress = false;
}
@Override
- public boolean maybeHandleCompletedReceive(ClientRequest req, long now, Struct body) {
- short apiKey = req.request().header().apiKey();
- if (apiKey == ApiKeys.METADATA.id && req.isInitiatedByNetworkClient()) {
- handleResponse(req.request().header(), body, now);
- return true;
- }
- return false;
+ public void handleCompletedMetadataResponse(RequestHeader requestHeader, long now, AbstractResponse response) {
+ if (!(response instanceof MetadataResponse))
+ throw new IllegalStateException("Unexpected response type in metadata handler: " + response);
+ handleMetadataResponse(requestHeader, (MetadataResponse) response, now);
}
@Override
@@ -592,9 +601,8 @@ public class NetworkClient implements KafkaClient {
this.metadata.requestUpdate();
}
- private void handleResponse(RequestHeader header, Struct body, long now) {
+ private void handleMetadataResponse(RequestHeader header, MetadataResponse response, long now) {
this.metadataFetchInProgress = false;
- MetadataResponse response = new MetadataResponse(body);
Cluster cluster = response.cluster();
// check if any topics metadata failed to get updated
Map<String, Errors> errors = response.errors();
@@ -612,14 +620,6 @@ public class NetworkClient implements KafkaClient {
}
/**
- * Create a metadata request for the given topics
- */
- private ClientRequest request(long now, String node, MetadataRequest metadata) {
- RequestSend send = new RequestSend(node, nextRequestHeader(ApiKeys.METADATA), metadata.toStruct());
- return new ClientRequest(now, true, send, null, true);
- }
-
- /**
* Return true if there's at least one connection establishment is currently underway
*/
private boolean isAnyNodeConnecting() {
@@ -644,9 +644,9 @@ public class NetworkClient implements KafkaClient {
metadataRequest = MetadataRequest.allTopics();
else
metadataRequest = new MetadataRequest(new ArrayList<>(metadata.topics()));
- ClientRequest clientRequest = request(now, nodeConnectionId, metadataRequest);
+
log.debug("Sending metadata request {} to node {}", metadataRequest, node.id());
- doSend(clientRequest, now);
+ sendInternalMetadataRequest(metadataRequest, nodeConnectionId, now);
return requestTimeoutMs;
}
@@ -674,4 +674,42 @@ public class NetworkClient implements KafkaClient {
}
+ static class InFlightRequest {
+ final RequestHeader header;
+ final String destination;
+ final RequestCompletionHandler callback;
+ final boolean expectResponse;
+ final boolean isInternalMetadataRequest; // used to flag metadata fetches which are triggered internally by NetworkClient
+ final Send send;
+ final long sendTimeMs;
+ final long createdTimeMs;
+
+ public InFlightRequest(RequestHeader header,
+ long createdTimeMs,
+ String destination,
+ RequestCompletionHandler callback,
+ boolean expectResponse,
+ boolean isInternalMetadataRequest,
+ Send send,
+ long sendTimeMs) {
+ this.header = header;
+ this.destination = destination;
+ this.callback = callback;
+ this.expectResponse = expectResponse;
+ this.isInternalMetadataRequest = isInternalMetadataRequest;
+ this.send = send;
+ this.sendTimeMs = sendTimeMs;
+ this.createdTimeMs = createdTimeMs;
+ }
+
+ public ClientResponse completed(AbstractResponse response, long timeMs) {
+ return new ClientResponse(header, callback, destination, createdTimeMs, timeMs, false, response);
+ }
+
+ public ClientResponse disconnected(long timeMs) {
+ return new ClientResponse(header, callback, destination, createdTimeMs, timeMs, true, null);
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 59319ef..c205273 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -393,17 +393,11 @@ public abstract class AbstractCoordinator implements Closeable {
}
private class JoinGroupResponseHandler extends CoordinatorResponseHandler<JoinGroupResponse, ByteBuffer> {
-
- @Override
- public JoinGroupResponse parse(ClientResponse response) {
- return new JoinGroupResponse(response.responseBody());
- }
-
@Override
public void handle(JoinGroupResponse joinResponse, RequestFuture<ByteBuffer> future) {
Errors error = Errors.forCode(joinResponse.errorCode());
if (error == Errors.NONE) {
- log.debug("Received successful join group response for group {}: {}", groupId, joinResponse.toStruct());
+ log.debug("Received successful join group response for group {}: {}", groupId, joinResponse);
sensors.joinLatency.record(response.requestLatencyMs());
synchronized (AbstractCoordinator.this) {
@@ -483,12 +477,6 @@ public abstract class AbstractCoordinator implements Closeable {
}
private class SyncGroupResponseHandler extends CoordinatorResponseHandler<SyncGroupResponse, ByteBuffer> {
-
- @Override
- public SyncGroupResponse parse(ClientResponse response) {
- return new SyncGroupResponse(response.responseBody());
- }
-
@Override
public void handle(SyncGroupResponse syncResponse,
RequestFuture<ByteBuffer> future) {
@@ -540,7 +528,7 @@ public abstract class AbstractCoordinator implements Closeable {
public void onSuccess(ClientResponse resp, RequestFuture<Void> future) {
log.debug("Received group coordinator response {}", resp);
- GroupCoordinatorResponse groupCoordinatorResponse = new GroupCoordinatorResponse(resp.responseBody());
+ GroupCoordinatorResponse groupCoordinatorResponse = (GroupCoordinatorResponse) resp.responseBody();
// use MAX_VALUE - node.id as the coordinator id to mimic separate connections
// for the coordinator in the underlying network client layer
// TODO: this needs to be better handled in KAFKA-1935
@@ -653,12 +641,6 @@ public abstract class AbstractCoordinator implements Closeable {
}
private class LeaveGroupResponseHandler extends CoordinatorResponseHandler<LeaveGroupResponse, Void> {
-
- @Override
- public LeaveGroupResponse parse(ClientResponse response) {
- return new LeaveGroupResponse(response.responseBody());
- }
-
@Override
public void handle(LeaveGroupResponse leaveResponse, RequestFuture<Void> future) {
Errors error = Errors.forCode(leaveResponse.errorCode());
@@ -680,12 +662,6 @@ public abstract class AbstractCoordinator implements Closeable {
}
private class HeartbeatResponseHandler extends CoordinatorResponseHandler<HeartbeatResponse, Void> {
-
- @Override
- public HeartbeatResponse parse(ClientResponse response) {
- return new HeartbeatResponse(response.responseBody());
- }
-
@Override
public void handle(HeartbeatResponse heartbeatResponse, RequestFuture<Void> future) {
sensors.heartbeatLatency.record(response.requestLatencyMs());
@@ -722,8 +698,6 @@ public abstract class AbstractCoordinator implements Closeable {
protected abstract class CoordinatorResponseHandler<R, T> extends RequestFutureAdapter<ClientResponse, T> {
protected ClientResponse response;
- public abstract R parse(ClientResponse response);
-
public abstract void handle(R response, RequestFuture<T> future);
@Override
@@ -735,10 +709,11 @@ public abstract class AbstractCoordinator implements Closeable {
}
@Override
+ @SuppressWarnings("unchecked")
public void onSuccess(ClientResponse clientResponse, RequestFuture<T> future) {
try {
this.response = clientResponse;
- R responseObj = parse(clientResponse);
+ R responseObj = (R) clientResponse.responseBody();
handle(responseObj, future);
} catch (RuntimeException e) {
if (!future.isDone())
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 2621c09..4889872 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -12,7 +12,6 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
@@ -624,11 +623,6 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
}
@Override
- public OffsetCommitResponse parse(ClientResponse response) {
- return new OffsetCommitResponse(response.responseBody());
- }
-
- @Override
public void handle(OffsetCommitResponse commitResponse, RequestFuture<Void> future) {
sensors.commitLatency.record(response.requestLatencyMs());
Set<String> unauthorizedTopics = new HashSet<>();
@@ -718,12 +712,6 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
}
private class OffsetFetchResponseHandler extends CoordinatorResponseHandler<OffsetFetchResponse, Map<TopicPartition, OffsetAndMetadata>> {
-
- @Override
- public OffsetFetchResponse parse(ClientResponse response) {
- return new OffsetFetchResponse(response.responseBody());
- }
-
@Override
public void handle(OffsetFetchResponse response, RequestFuture<Map<TopicPartition, OffsetAndMetadata>> future) {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>(response.responseData().size());
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClient.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClient.java
index 2495b23..d9baa56 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkClient.java
@@ -25,7 +25,6 @@ import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.RequestSend;
import org.apache.kafka.common.utils.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,14 +97,14 @@ public class ConsumerNetworkClient implements Closeable {
}
private RequestFuture<ClientResponse> send(Node node,
- ApiKeys api,
- short version,
- AbstractRequest request) {
+ ApiKeys api,
+ short version,
+ AbstractRequest request) {
long now = time.milliseconds();
RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler();
RequestHeader header = client.nextRequestHeader(api, version);
- RequestSend send = new RequestSend(node.idString(), header, request.toStruct());
- put(node, new ClientRequest(now, true, send, completionHandler));
+ ClientRequest clientRequest = new ClientRequest(node.idString(), now, true, header, request, completionHandler);
+ put(node, clientRequest);
// wakeup the client in case it is blocking in poll so that we can send the queued request
client.wakeup();
@@ -336,9 +335,9 @@ public class ConsumerNetworkClient implements Closeable {
// coordinator failures traversing the unsent list again.
iterator.remove();
for (ClientRequest request : requestEntry.getValue()) {
- RequestFutureCompletionHandler handler =
- (RequestFutureCompletionHandler) request.callback();
- handler.onComplete(new ClientResponse(request, now, true, null));
+ RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback();
+ handler.onComplete(new ClientResponse(request.header(), request.callback(), request.destination(),
+ request.createdTimeMs(), now, true, null));
}
}
}
@@ -369,8 +368,8 @@ public class ConsumerNetworkClient implements Closeable {
synchronized (this) {
List<ClientRequest> unsentRequests = unsent.remove(node);
if (unsentRequests != null) {
- for (ClientRequest request : unsentRequests) {
- RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback();
+ for (ClientRequest unsentRequest : unsentRequests) {
+ RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) unsentRequest.callback();
handler.onFailure(e);
}
}
@@ -468,12 +467,11 @@ public class ConsumerNetworkClient implements Closeable {
if (e != null) {
future.raise(e);
} else if (response.wasDisconnected()) {
- ClientRequest request = response.request();
- RequestSend send = request.request();
- ApiKeys api = ApiKeys.forId(send.header().apiKey());
- int correlation = send.header().correlationId();
+ RequestHeader requestHeader = response.requestHeader();
+ ApiKeys api = ApiKeys.forId(requestHeader.apiKey());
+ int correlation = requestHeader.correlationId();
log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",
- api, request, correlation, send.destination());
+ api, requestHeader, correlation, response.destination());
future.raise(DisconnectException.INSTANCE);
} else {
future.complete(response);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index bfc1a0b..703ea29 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -41,7 +41,6 @@ import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.record.InvalidRecordException;
import org.apache.kafka.common.record.LogEntry;
-import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.record.TimestampType;
import org.apache.kafka.common.requests.FetchRequest;
@@ -155,7 +154,7 @@ public class Fetcher<K, V> {
.addListener(new RequestFutureListener<ClientResponse>() {
@Override
public void onSuccess(ClientResponse resp) {
- FetchResponse response = new FetchResponse(resp.responseBody());
+ FetchResponse response = (FetchResponse) resp.responseBody();
if (!matchesRequestedPartitions(request, response)) {
// obviously we expect the broker to always send us valid responses, so this check
// is mainly for test cases where mock fetch responses must be manually crafted.
@@ -256,7 +255,7 @@ public class Fetcher<K, V> {
throw future.exception();
if (future.succeeded()) {
- MetadataResponse response = new MetadataResponse(future.value().responseBody());
+ MetadataResponse response = (MetadataResponse) future.value().responseBody();
Cluster cluster = response.cluster();
Set<String> unauthorizedTopics = cluster.unauthorizedTopics();
@@ -549,7 +548,7 @@ public class Fetcher<K, V> {
.compose(new RequestFutureAdapter<ClientResponse, Map<TopicPartition, OffsetAndTimestamp>>() {
@Override
public void onSuccess(ClientResponse response, RequestFuture<Map<TopicPartition, OffsetAndTimestamp>> future) {
- ListOffsetResponse lor = new ListOffsetResponse(response.responseBody());
+ ListOffsetResponse lor = (ListOffsetResponse) response.responseBody();
log.trace("Received ListOffsetResponse {} from broker {}", lor, node);
handleListOffsetResponse(timestampsToSearch, lor, future);
}
@@ -673,10 +672,8 @@ public class Fetcher<K, V> {
return null;
}
- ByteBuffer buffer = partition.recordSet;
- MemoryRecords records = MemoryRecords.readableRecords(buffer);
List<ConsumerRecord<K, V>> parsed = new ArrayList<>();
- for (LogEntry logEntry : records) {
+ for (LogEntry logEntry : partition.records) {
// Skip the messages earlier than current position.
if (logEntry.offset() >= position) {
parsed.add(parseRecord(tp, logEntry));
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
index c71bb67..7555b71 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
@@ -12,20 +12,13 @@
*/
package org.apache.kafka.clients.producer.internals;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.KafkaClient;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.RequestCompletionHandler;
import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.InvalidMetadataException;
@@ -34,7 +27,6 @@ import org.apache.kafka.common.errors.TopicAuthorizationException;
import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
import org.apache.kafka.common.metrics.Measurable;
import org.apache.kafka.common.metrics.MetricConfig;
-import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.metrics.Sensor;
import org.apache.kafka.common.metrics.stats.Avg;
@@ -42,15 +34,22 @@ import org.apache.kafka.common.metrics.stats.Max;
import org.apache.kafka.common.metrics.stats.Rate;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.requests.ProduceRequest;
import org.apache.kafka.common.requests.ProduceResponse;
-import org.apache.kafka.common.requests.RequestSend;
+import org.apache.kafka.common.requests.RequestHeader;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
/**
* The background thread that handles the sending of produce requests to the Kafka cluster. This thread makes metadata
* requests to renew its view of the cluster and then sends produce requests to the appropriate nodes.
@@ -210,19 +209,17 @@ public class Sender implements Runnable {
this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount);
sensors.updateProduceRequestMetrics(batches);
- List<ClientRequest> requests = createProduceRequests(batches, now);
+
// If we have any nodes that are ready to send + have sendable data, poll with 0 timeout so this can immediately
// loop and try sending more data. Otherwise, the timeout is determined by nodes that have partitions with data
// that isn't yet sendable (e.g. lingering, backing off). Note that this specifically does not include nodes
// with sendable data that aren't ready to send since they would cause busy looping.
long pollTimeout = Math.min(result.nextReadyCheckDelayMs, notReadyTimeout);
- if (result.readyNodes.size() > 0) {
+ if (!result.readyNodes.isEmpty()) {
log.trace("Nodes with data ready to send: {}", result.readyNodes);
- log.trace("Created {} produce requests: {}", requests.size(), requests);
pollTimeout = 0;
}
- for (ClientRequest request : requests)
- client.send(request, now);
+ sendProduceRequests(batches, now);
// if some partitions are already ready to be sent, the select time would be 0;
// otherwise if some partition already has some data accumulated but not ready yet,
@@ -254,20 +251,16 @@ public class Sender implements Runnable {
* Handle a produce response
*/
private void handleProduceResponse(ClientResponse response, Map<TopicPartition, RecordBatch> batches, long now) {
- int correlationId = response.request().request().header().correlationId();
+ int correlationId = response.requestHeader().correlationId();
if (response.wasDisconnected()) {
- log.trace("Cancelled request {} due to node {} being disconnected", response, response.request()
- .request()
- .destination());
+ log.trace("Cancelled request {} due to node {} being disconnected", response, response.destination());
for (RecordBatch batch : batches.values())
completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, Record.NO_TIMESTAMP, correlationId, now);
} else {
- log.trace("Received produce response from node {} with correlation id {}",
- response.request().request().destination(),
- correlationId);
+ log.trace("Received produce response from node {} with correlation id {}", response.destination(), correlationId);
// if we have a response, parse it
if (response.hasResponse()) {
- ProduceResponse produceResponse = new ProduceResponse(response.responseBody());
+ ProduceResponse produceResponse = (ProduceResponse) response.responseBody();
for (Map.Entry<TopicPartition, ProduceResponse.PartitionResponse> entry : produceResponse.responses().entrySet()) {
TopicPartition tp = entry.getKey();
ProduceResponse.PartitionResponse partResp = entry.getValue();
@@ -275,7 +268,7 @@ public class Sender implements Runnable {
RecordBatch batch = batches.get(tp);
completeBatch(batch, error, partResp.baseOffset, partResp.timestamp, correlationId, now);
}
- this.sensors.recordLatency(response.request().request().destination(), response.requestLatencyMs());
+ this.sensors.recordLatency(response.destination(), response.requestLatencyMs());
this.sensors.recordThrottleTime(produceResponse.getThrottleTime());
} else {
// this is the acks = 0 case, just complete all requests
@@ -339,35 +332,36 @@ public class Sender implements Runnable {
/**
* Transfer the record batches into a list of produce requests on a per-node basis
*/
- private List<ClientRequest> createProduceRequests(Map<Integer, List<RecordBatch>> collated, long now) {
- List<ClientRequest> requests = new ArrayList<ClientRequest>(collated.size());
+ private void sendProduceRequests(Map<Integer, List<RecordBatch>> collated, long now) {
for (Map.Entry<Integer, List<RecordBatch>> entry : collated.entrySet())
- requests.add(produceRequest(now, entry.getKey(), acks, requestTimeout, entry.getValue()));
- return requests;
+ sendProduceRequest(now, entry.getKey(), acks, requestTimeout, entry.getValue());
}
/**
* Create a produce request from the given record batches
*/
- private ClientRequest produceRequest(long now, int destination, short acks, int timeout, List<RecordBatch> batches) {
- Map<TopicPartition, ByteBuffer> produceRecordsByPartition = new HashMap<TopicPartition, ByteBuffer>(batches.size());
- final Map<TopicPartition, RecordBatch> recordsByPartition = new HashMap<TopicPartition, RecordBatch>(batches.size());
+ private void sendProduceRequest(long now, int destination, short acks, int timeout, List<RecordBatch> batches) {
+ Map<TopicPartition, MemoryRecords> produceRecordsByPartition = new HashMap<>(batches.size());
+ final Map<TopicPartition, RecordBatch> recordsByPartition = new HashMap<>(batches.size());
for (RecordBatch batch : batches) {
TopicPartition tp = batch.topicPartition;
- produceRecordsByPartition.put(tp, batch.records.buffer());
+ produceRecordsByPartition.put(tp, batch.records);
recordsByPartition.put(tp, batch);
}
- ProduceRequest request = new ProduceRequest(acks, timeout, produceRecordsByPartition);
- RequestSend send = new RequestSend(Integer.toString(destination),
- this.client.nextRequestHeader(ApiKeys.PRODUCE),
- request.toStruct());
+
+ ProduceRequest produceRequest = new ProduceRequest(acks, timeout, produceRecordsByPartition);
+ RequestHeader header = this.client.nextRequestHeader(ApiKeys.PRODUCE);
RequestCompletionHandler callback = new RequestCompletionHandler() {
public void onComplete(ClientResponse response) {
handleProduceResponse(response, recordsByPartition, time.milliseconds());
}
};
- return new ClientRequest(now, acks != 0, send, callback);
+ String nodeId = Integer.toString(destination);
+ ClientRequest clientRequest = new ClientRequest(nodeId, now, acks != 0, header, produceRequest, callback);
+
+ client.send(clientRequest, now);
+ log.trace("Sent produce request to {}: {}", nodeId, produceRequest);
}
/**
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java
index 9e213ec..3683283 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java
@@ -29,11 +29,10 @@ public class ByteBufferSend implements Send {
private boolean pending = false;
public ByteBufferSend(String destination, ByteBuffer... buffers) {
- super();
this.destination = destination;
this.buffers = buffers;
- for (int i = 0; i < buffers.length; i++)
- remaining += buffers[i].remaining();
+ for (ByteBuffer buffer : buffers)
+ remaining += buffer.remaining();
this.size = remaining;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
index 49964b0..5e4bf2c 100644
--- a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
+++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java
@@ -29,8 +29,8 @@ public class NetworkSend extends ByteBufferSend {
private static ByteBuffer[] sizeDelimit(ByteBuffer[] buffers) {
int size = 0;
- for (int i = 0; i < buffers.length; i++)
- size += buffers[i].remaining();
+ for (ByteBuffer buffer : buffers)
+ size += buffer.remaining();
ByteBuffer[] delimited = new ByteBuffer[buffers.length + 1];
delimited[0] = ByteBuffer.allocate(4);
delimited[0].putInt(size);
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
index 6178b80..e07c3c3 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java
@@ -63,7 +63,7 @@ public enum ApiKeys {
/** an english description of the api--this is for debugging and can change */
public final String name;
- private ApiKeys(int id, String name) {
+ ApiKeys(int id, String name) {
this.id = (short) id;
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
index 9e21f3b..cd4e6e3 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java
@@ -31,6 +31,7 @@ import static org.apache.kafka.common.protocol.types.Type.BYTES;
import static org.apache.kafka.common.protocol.types.Type.INT16;
import static org.apache.kafka.common.protocol.types.Type.INT32;
import static org.apache.kafka.common.protocol.types.Type.INT64;
+import static org.apache.kafka.common.protocol.types.Type.RECORDS;
import static org.apache.kafka.common.protocol.types.Type.STRING;
import static org.apache.kafka.common.protocol.types.Type.NULLABLE_STRING;
@@ -135,7 +136,7 @@ public class Protocol {
public static final Schema TOPIC_PRODUCE_DATA_V0 = new Schema(new Field("topic", STRING),
new Field("data", new ArrayOf(new Schema(new Field("partition", INT32),
- new Field("record_set", BYTES)))));
+ new Field("record_set", RECORDS)))));
public static final Schema PRODUCE_REQUEST_V0 = new Schema(new Field("acks",
INT16,
@@ -500,14 +501,15 @@ public class Protocol {
new ArrayOf(FETCH_REQUEST_TOPIC_V0),
"Topics to fetch in the order provided."));
- public static final Schema FETCH_RESPONSE_PARTITION_V0 = new Schema(new Field("partition",
- INT32,
- "Topic partition id."),
- new Field("error_code", INT16),
- new Field("high_watermark",
- INT64,
- "Last committed offset."),
- new Field("record_set", BYTES));
+ public static final Schema FETCH_RESPONSE_PARTITION_HEADER_V0 = new Schema(new Field("partition",
+ INT32,
+ "Topic partition id."),
+ new Field("error_code", INT16),
+ new Field("high_watermark",
+ INT64,
+ "Last committed offset."));
+ public static final Schema FETCH_RESPONSE_PARTITION_V0 = new Schema(new Field("partition_header", FETCH_RESPONSE_PARTITION_HEADER_V0),
+ new Field("record_set", RECORDS));
public static final Schema FETCH_RESPONSE_TOPIC_V0 = new Schema(new Field("topic", STRING),
new Field("partition_responses",
@@ -515,6 +517,7 @@ public class Protocol {
public static final Schema FETCH_RESPONSE_V0 = new Schema(new Field("responses",
new ArrayOf(FETCH_RESPONSE_TOPIC_V0)));
+
public static final Schema FETCH_RESPONSE_V1 = new Schema(new Field("throttle_time_ms",
INT32,
"Duration in milliseconds for which the request was throttled" +
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
index 207f108..1ba8e44 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/ArrayOf.java
@@ -54,8 +54,9 @@ public class ArrayOf extends Type {
Object[] objs = (Object[]) o;
int size = objs.length;
buffer.putInt(size);
- for (int i = 0; i < size; i++)
- type.write(buffer, objs[i]);
+
+ for (Object obj : objs)
+ type.write(buffer, obj);
}
@Override
@@ -81,8 +82,8 @@ public class ArrayOf extends Type {
return size;
Object[] objs = (Object[]) o;
- for (int i = 0; i < objs.length; i++)
- size += type.sizeOf(objs[i]);
+ for (Object obj : objs)
+ size += type.sizeOf(obj);
return size;
}
@@ -102,8 +103,8 @@ public class ArrayOf extends Type {
return null;
Object[] array = (Object[]) item;
- for (int i = 0; i < array.length; i++)
- type.validate(array[i]);
+ for (Object obj : array)
+ type.validate(obj);
return array;
} catch (ClassCastException e) {
throw new SchemaException("Not an Object[].");
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
index e8dce31..efbfd14 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Schema.java
@@ -31,7 +31,7 @@ public class Schema extends Type {
*/
public Schema(Field... fs) {
this.fields = new Field[fs.length];
- this.fieldsByName = new HashMap<String, Field>();
+ this.fieldsByName = new HashMap<>();
for (int i = 0; i < this.fields.length; i++) {
Field field = fs[i];
if (fieldsByName.containsKey(field.name))
@@ -47,13 +47,12 @@ public class Schema extends Type {
@Override
public void write(ByteBuffer buffer, Object o) {
Struct r = (Struct) o;
- for (int i = 0; i < fields.length; i++) {
- Field f = fields[i];
+ for (Field field : fields) {
try {
- Object value = f.type().validate(r.get(f));
- f.type.write(buffer, value);
+ Object value = field.type().validate(r.get(field));
+ field.type.write(buffer, value);
} catch (Exception e) {
- throw new SchemaException("Error writing field '" + f.name +
+ throw new SchemaException("Error writing field '" + field.name +
"': " +
(e.getMessage() == null ? e.getClass().getName() : e.getMessage()));
}
@@ -85,8 +84,8 @@ public class Schema extends Type {
public int sizeOf(Object o) {
int size = 0;
Struct r = (Struct) o;
- for (int i = 0; i < fields.length; i++)
- size += fields[i].type.sizeOf(r.get(fields[i]));
+ for (Field field : fields)
+ size += field.type.sizeOf(r.get(field));
return size;
}
@@ -146,8 +145,7 @@ public class Schema extends Type {
public Struct validate(Object item) {
try {
Struct struct = (Struct) item;
- for (int i = 0; i < this.fields.length; i++) {
- Field field = this.fields[i];
+ for (Field field : fields) {
try {
field.type.validate(struct.get(field));
} catch (SchemaException e) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
index 212d701..0165ce6 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java
@@ -12,6 +12,8 @@
*/
package org.apache.kafka.common.protocol.types;
+import org.apache.kafka.common.record.Records;
+
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -108,6 +110,10 @@ public class Struct {
return (Byte) get(name);
}
+ public Records getRecords(String name) {
+ return (Records) get(name);
+ }
+
public Short getShort(Field field) {
return (Short) get(field);
}
@@ -272,16 +278,6 @@ public class Struct {
this.schema.validate(this);
}
- /**
- * Create a byte buffer containing the serialized form of the values in this struct. This method can choose to break
- * the struct into multiple ByteBuffers if need be.
- */
- public ByteBuffer[] toBytes() {
- ByteBuffer buffer = ByteBuffer.allocate(sizeOf());
- writeTo(buffer);
- return new ByteBuffer[] {buffer};
- }
-
@Override
public String toString() {
StringBuilder b = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
index 43b4a37..e2f7baf 100644
--- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
+++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Type.java
@@ -18,6 +18,9 @@ package org.apache.kafka.common.protocol.types;
import java.nio.ByteBuffer;
+import org.apache.kafka.common.record.FileRecords;
+import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.Records;
import org.apache.kafka.common.utils.Utils;
/**
@@ -424,5 +427,50 @@ public abstract class Type {
}
};
+ public static final Type RECORDS = new Type() {
+ @Override
+ public boolean isNullable() {
+ return true;
+ }
+
+ @Override
+ public void write(ByteBuffer buffer, Object o) {
+ if (o instanceof FileRecords)
+ throw new IllegalArgumentException("FileRecords must be written to the channel directly");
+ MemoryRecords records = (MemoryRecords) o;
+ NULLABLE_BYTES.write(buffer, records.buffer().duplicate());
+ }
+
+ @Override
+ public Object read(ByteBuffer buffer) {
+ ByteBuffer recordsBuffer = (ByteBuffer) NULLABLE_BYTES.read(buffer);
+ return MemoryRecords.readableRecords(recordsBuffer);
+ }
+
+ @Override
+ public int sizeOf(Object o) {
+ if (o == null)
+ return 4;
+
+ Records records = (Records) o;
+ return 4 + records.sizeInBytes();
+ }
+
+ @Override
+ public String toString() {
+ return "RECORDS";
+ }
+
+ @Override
+ public Records validate(Object item) {
+ if (item == null)
+ return null;
+
+ if (item instanceof Records)
+ return (Records) item;
+
+ throw new SchemaException(item + " is not an instance of " + Records.class.getName());
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
new file mode 100644
index 0000000..bdae08d
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/common/record/FileRecords.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+package org.apache.kafka.common.record;
+
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.network.TransportLayer;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.GatheringByteChannel;
+
+/**
+ * File-backed record set.
+ */
+public class FileRecords implements Records {
+ private final File file;
+ private final FileChannel channel;
+ private final long start;
+ private final long end;
+ private final long size;
+
+ public FileRecords(File file,
+ FileChannel channel,
+ int start,
+ int end,
+ boolean isSlice) throws IOException {
+ this.file = file;
+ this.channel = channel;
+ this.start = start;
+ this.end = end;
+
+ if (isSlice)
+ this.size = end - start;
+ else
+ this.size = Math.min(channel.size(), end) - start;
+ }
+
+ @Override
+ public int sizeInBytes() {
+ return (int) size;
+ }
+
+ @Override
+ public long writeTo(GatheringByteChannel destChannel, long offset, int length) throws IOException {
+ long newSize = Math.min(channel.size(), end) - start;
+ if (newSize < size)
+ throw new KafkaException(String.format("Size of FileRecords %s has been truncated during write: old size %d, new size %d", file.getAbsolutePath(), size, newSize));
+
+ if (offset > size)
+ throw new KafkaException(String.format("The requested offset %d is out of range. The size of this FileRecords is %d.", offset, size));
+
+ long position = start + offset;
+ long count = Math.min(length, this.size - offset);
+ if (destChannel instanceof TransportLayer) {
+ TransportLayer tl = (TransportLayer) destChannel;
+ return tl.transferFrom(this.channel, position, count);
+ } else {
+ return this.channel.transferTo(position, count, destChannel);
+ }
+ }
+
+ @Override
+ public RecordsIterator iterator() {
+ return new RecordsIterator(new FileLogInputStream(channel, start, end), false);
+ }
+
+ private static class FileLogInputStream implements LogInputStream {
+ private long position;
+ protected final long end;
+ protected final FileChannel channel;
+ private final ByteBuffer logHeaderBuffer = ByteBuffer.allocate(Records.LOG_OVERHEAD);
+
+ public FileLogInputStream(FileChannel channel, long start, long end) {
+ this.channel = channel;
+ this.position = start;
+ this.end = end;
+ }
+
+ @Override
+ public LogEntry nextEntry() throws IOException {
+ if (position + Records.LOG_OVERHEAD >= end)
+ return null;
+
+ logHeaderBuffer.rewind();
+ channel.read(logHeaderBuffer, position);
+ if (logHeaderBuffer.hasRemaining())
+ return null;
+
+ logHeaderBuffer.rewind();
+ long offset = logHeaderBuffer.getLong();
+ int size = logHeaderBuffer.getInt();
+ if (size < 0)
+ throw new IllegalStateException("Record with size " + size);
+
+ if (position + Records.LOG_OVERHEAD + size > end)
+ return null;
+
+ ByteBuffer recordBuffer = ByteBuffer.allocate(size);
+ channel.read(recordBuffer, position + Records.LOG_OVERHEAD);
+ if (recordBuffer.hasRemaining())
+ return null;
+ recordBuffer.rewind();
+
+ Record record = new Record(recordBuffer);
+ LogEntry logEntry = new LogEntry(offset, record);
+ position += logEntry.size();
+ return logEntry;
+ }
+ }
+}
[2/5] kafka git commit: KAFKA-2066;
Use client-side FetchRequest/FetchResponse on server
Posted by jg...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index b7645dd..097b4fc 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -12,31 +12,19 @@
*/
package org.apache.kafka.clients.producer.internals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.internals.ClusterResourceListeners;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.ClusterResourceListeners;
import org.apache.kafka.common.metrics.KafkaMetric;
import org.apache.kafka.common.metrics.MetricConfig;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.requests.ProduceResponse;
@@ -46,6 +34,17 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class SenderTest {
private static final int MAX_REQUEST_SIZE = 1024 * 1024;
@@ -69,7 +68,7 @@ public class SenderTest {
@Before
public void setup() {
- Map<String, String> metricTags = new LinkedHashMap<String, String>();
+ Map<String, String> metricTags = new LinkedHashMap<>();
metricTags.put("client-id", CLIENT_ID);
MetricConfig metricConfig = new MetricConfig().tags(metricTags);
metrics = new Metrics(metricConfig, time);
@@ -147,7 +146,7 @@ public class SenderTest {
Future<RecordMetadata> future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
sender.run(time.milliseconds()); // connect
sender.run(time.milliseconds()); // send produce request
- String id = client.requests().peek().request().destination();
+ String id = client.requests().peek().destination();
Node node = new Node(Integer.valueOf(id), "localhost", 0);
assertEquals(1, client.inFlightRequestCount());
assertTrue("Client ready status should be true", client.isReady(node, 0L));
@@ -168,7 +167,7 @@ public class SenderTest {
future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
sender.run(time.milliseconds()); // send produce request
for (int i = 0; i < maxRetries + 1; i++) {
- client.disconnect(client.requests().peek().request().destination());
+ client.disconnect(client.requests().peek().destination());
sender.run(time.milliseconds()); // receive error
sender.run(time.milliseconds()); // reconnect
sender.run(time.milliseconds()); // resend
@@ -205,8 +204,8 @@ public class SenderTest {
accumulator.append(tp2, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT);
sender.run(time.milliseconds()); // connect
sender.run(time.milliseconds()); // send produce request
- String id = client.requests().peek().request().destination();
- assertEquals(ApiKeys.PRODUCE.id, client.requests().peek().request().header().apiKey());
+ String id = client.requests().peek().destination();
+ assertEquals(ApiKeys.PRODUCE.id, client.requests().peek().header().apiKey());
Node node = new Node(Integer.valueOf(id), "localhost", 0);
assertEquals(1, client.inFlightRequestCount());
assertTrue("Client ready status should be true", client.isReady(node, 0L));
@@ -272,11 +271,10 @@ public class SenderTest {
}
}
- private Struct produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) {
+ private ProduceResponse produceResponse(TopicPartition tp, long offset, int error, int throttleTimeMs) {
ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse((short) error, offset, Record.NO_TIMESTAMP);
Map<TopicPartition, ProduceResponse.PartitionResponse> partResp = Collections.singletonMap(tp, resp);
- ProduceResponse response = new ProduceResponse(partResp, throttleTimeMs);
- return response.toStruct();
+ return new ProduceResponse(partResp, throttleTimeMs);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
index 30faac1..a5eb22a 100644
--- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java
@@ -16,15 +16,20 @@ package org.apache.kafka.common.requests;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.UnknownServerException;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.ProtoUtils;
import org.apache.kafka.common.protocol.SecurityProtocol;
+import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.record.MemoryRecords;
import org.apache.kafka.common.record.Record;
import org.junit.Test;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
+import java.nio.channels.GatheringByteChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -36,6 +41,7 @@ import java.util.Map;
import java.util.Set;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class RequestResponseTest {
@@ -113,7 +119,6 @@ public class RequestResponseTest {
for (AbstractRequestResponse req : requestResponseList)
checkSerialization(req, null);
-
checkOlderFetchVersions();
checkSerialization(createMetadataResponse(0), 0);
checkSerialization(createMetadataResponse(1), 1);
@@ -180,7 +185,9 @@ public class RequestResponseTest {
@Test
public void fetchResponseVersionTest() {
Map<TopicPartition, FetchResponse.PartitionData> responseData = new HashMap<>();
- responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE.code(), 1000000, ByteBuffer.allocate(10)));
+
+ MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10));
+ responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE.code(), 1000000, records));
FetchResponse v0Response = new FetchResponse(responseData);
FetchResponse v1Response = new FetchResponse(responseData, 10);
@@ -193,6 +200,34 @@ public class RequestResponseTest {
}
@Test
+ public void verifyFetchResponseFullWrite() throws Exception {
+ FetchResponse fetchResponse = createFetchResponse();
+ RequestHeader header = new RequestHeader(ApiKeys.FETCH.id, "client", 15);
+
+ Send send = fetchResponse.toSend("1", header);
+ ByteBufferChannel channel = new ByteBufferChannel(send.size());
+ send.writeTo(channel);
+ channel.close();
+
+ ByteBuffer buf = channel.buf;
+
+ // read the size
+ int size = buf.getInt();
+ assertTrue(size > 0);
+
+ // read the header
+ ResponseHeader responseHeader = ResponseHeader.parse(channel.buf);
+ assertEquals(header.correlationId(), responseHeader.correlationId());
+
+ // read the body
+ Struct responseBody = ProtoUtils.responseSchema(ApiKeys.FETCH.id, header.apiVersion()).read(buf);
+ FetchResponse parsedResponse = new FetchResponse(responseBody);
+ assertEquals(parsedResponse, fetchResponse);
+
+ assertEquals(size, responseHeader.sizeOf() + parsedResponse.sizeOf());
+ }
+
+ @Test
public void testControlledShutdownResponse() {
ControlledShutdownResponse response = createControlledShutdownResponse();
ByteBuffer buffer = ByteBuffer.allocate(response.sizeOf());
@@ -216,24 +251,24 @@ public class RequestResponseTest {
assertEquals("", deserialized.clientId()); // null is defaulted to ""
}
- private AbstractRequestResponse createRequestHeader() {
+ private RequestHeader createRequestHeader() {
return new RequestHeader((short) 10, (short) 1, "", 10);
}
- private AbstractRequestResponse createResponseHeader() {
+ private ResponseHeader createResponseHeader() {
return new ResponseHeader(10);
}
- private AbstractRequest createGroupCoordinatorRequest() {
+ private GroupCoordinatorRequest createGroupCoordinatorRequest() {
return new GroupCoordinatorRequest("test-group");
}
- private AbstractRequestResponse createGroupCoordinatorResponse() {
+ private GroupCoordinatorResponse createGroupCoordinatorResponse() {
return new GroupCoordinatorResponse(Errors.NONE.code(), new Node(10, "host1", 2014));
}
@SuppressWarnings("deprecation")
- private AbstractRequest createFetchRequest(int version) {
+ private FetchRequest createFetchRequest(int version) {
LinkedHashMap<TopicPartition, FetchRequest.PartitionData> fetchData = new LinkedHashMap<>();
fetchData.put(new TopicPartition("test1", 0), new FetchRequest.PartitionData(100, 1000000));
fetchData.put(new TopicPartition("test2", 0), new FetchRequest.PartitionData(200, 1000000));
@@ -243,22 +278,23 @@ public class RequestResponseTest {
return new FetchRequest(100, 1000, 1000000, fetchData);
}
- private AbstractRequestResponse createFetchResponse() {
+ private FetchResponse createFetchResponse() {
Map<TopicPartition, FetchResponse.PartitionData> responseData = new HashMap<>();
- responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE.code(), 1000000, ByteBuffer.allocate(10)));
- return new FetchResponse(responseData, 0);
+ MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10));
+ responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData(Errors.NONE.code(), 1000000, records));
+ return new FetchResponse(responseData, 25);
}
- private AbstractRequest createHeartBeatRequest() {
+ private HeartbeatRequest createHeartBeatRequest() {
return new HeartbeatRequest("group1", 1, "consumer1");
}
- private AbstractRequestResponse createHeartBeatResponse() {
+ private HeartbeatResponse createHeartBeatResponse() {
return new HeartbeatResponse(Errors.NONE.code());
}
@SuppressWarnings("deprecation")
- private AbstractRequest createJoinGroupRequest(int version) {
+ private JoinGroupRequest createJoinGroupRequest(int version) {
ByteBuffer metadata = ByteBuffer.wrap(new byte[] {});
List<JoinGroupRequest.ProtocolMetadata> protocols = new ArrayList<>();
protocols.add(new JoinGroupRequest.ProtocolMetadata("consumer-range", metadata));
@@ -269,27 +305,27 @@ public class RequestResponseTest {
}
}
- private AbstractRequestResponse createJoinGroupResponse() {
+ private JoinGroupResponse createJoinGroupResponse() {
Map<String, ByteBuffer> members = new HashMap<>();
members.put("consumer1", ByteBuffer.wrap(new byte[]{}));
members.put("consumer2", ByteBuffer.wrap(new byte[]{}));
return new JoinGroupResponse(Errors.NONE.code(), 1, "range", "consumer1", "leader", members);
}
- private AbstractRequest createListGroupsRequest() {
+ private ListGroupsRequest createListGroupsRequest() {
return new ListGroupsRequest();
}
- private AbstractRequestResponse createListGroupsResponse() {
+ private ListGroupsResponse createListGroupsResponse() {
List<ListGroupsResponse.Group> groups = Arrays.asList(new ListGroupsResponse.Group("test-group", "consumer"));
return new ListGroupsResponse(Errors.NONE.code(), groups);
}
- private AbstractRequest createDescribeGroupRequest() {
+ private DescribeGroupsRequest createDescribeGroupRequest() {
return new DescribeGroupsRequest(Collections.singletonList("test-group"));
}
- private AbstractRequestResponse createDescribeGroupResponse() {
+ private DescribeGroupsResponse createDescribeGroupResponse() {
String clientId = "consumer-1";
String clientHost = "localhost";
ByteBuffer empty = ByteBuffer.allocate(0);
@@ -300,16 +336,16 @@ public class RequestResponseTest {
return new DescribeGroupsResponse(Collections.singletonMap("test-group", metadata));
}
- private AbstractRequest createLeaveGroupRequest() {
+ private LeaveGroupRequest createLeaveGroupRequest() {
return new LeaveGroupRequest("group1", "consumer1");
}
- private AbstractRequestResponse createLeaveGroupResponse() {
+ private LeaveGroupResponse createLeaveGroupResponse() {
return new LeaveGroupResponse(Errors.NONE.code());
}
@SuppressWarnings("deprecation")
- private AbstractRequest createListOffsetRequest(int version) {
+ private ListOffsetRequest createListOffsetRequest(int version) {
if (version == 0) {
Map<TopicPartition, ListOffsetRequest.PartitionData> offsetData = new HashMap<>();
offsetData.put(new TopicPartition("test", 0), new ListOffsetRequest.PartitionData(1000000L, 10));
@@ -324,7 +360,7 @@ public class RequestResponseTest {
}
@SuppressWarnings("deprecation")
- private AbstractRequestResponse createListOffsetResponse(int version) {
+ private ListOffsetResponse createListOffsetResponse(int version) {
if (version == 0) {
Map<TopicPartition, ListOffsetResponse.PartitionData> responseData = new HashMap<>();
responseData.put(new TopicPartition("test", 0), new ListOffsetResponse.PartitionData(Errors.NONE.code(), Arrays.asList(100L)));
@@ -338,11 +374,11 @@ public class RequestResponseTest {
}
}
- private AbstractRequest createMetadataRequest(List<String> topics) {
+ private MetadataRequest createMetadataRequest(List<String> topics) {
return new MetadataRequest(topics);
}
- private AbstractRequestResponse createMetadataResponse(int version) {
+ private MetadataResponse createMetadataResponse(int version) {
Node node = new Node(1, "host1", 1001);
List<Node> replicas = Arrays.asList(node);
List<Node> isr = Arrays.asList(node);
@@ -357,7 +393,7 @@ public class RequestResponseTest {
}
@SuppressWarnings("deprecation")
- private AbstractRequest createOffsetCommitRequest(int version) {
+ private OffsetCommitRequest createOffsetCommitRequest(int version) {
Map<TopicPartition, OffsetCommitRequest.PartitionData> commitData = new HashMap<>();
commitData.put(new TopicPartition("test", 0), new OffsetCommitRequest.PartitionData(100, ""));
commitData.put(new TopicPartition("test", 1), new OffsetCommitRequest.PartitionData(200, null));
@@ -371,47 +407,47 @@ public class RequestResponseTest {
throw new IllegalArgumentException("Unknown offset commit request version " + version);
}
- private AbstractRequestResponse createOffsetCommitResponse() {
+ private OffsetCommitResponse createOffsetCommitResponse() {
Map<TopicPartition, Short> responseData = new HashMap<>();
responseData.put(new TopicPartition("test", 0), Errors.NONE.code());
return new OffsetCommitResponse(responseData);
}
- private AbstractRequest createOffsetFetchRequest() {
+ private OffsetFetchRequest createOffsetFetchRequest() {
return new OffsetFetchRequest("group1", Arrays.asList(new TopicPartition("test11", 1)));
}
- private AbstractRequestResponse createOffsetFetchResponse() {
+ private OffsetFetchResponse createOffsetFetchResponse() {
Map<TopicPartition, OffsetFetchResponse.PartitionData> responseData = new HashMap<>();
responseData.put(new TopicPartition("test", 0), new OffsetFetchResponse.PartitionData(100L, "", Errors.NONE.code()));
responseData.put(new TopicPartition("test", 1), new OffsetFetchResponse.PartitionData(100L, null, Errors.NONE.code()));
return new OffsetFetchResponse(responseData);
}
- private AbstractRequest createProduceRequest() {
- Map<TopicPartition, ByteBuffer> produceData = new HashMap<>();
- produceData.put(new TopicPartition("test", 0), ByteBuffer.allocate(10));
+ private ProduceRequest createProduceRequest() {
+ Map<TopicPartition, MemoryRecords> produceData = new HashMap<>();
+ produceData.put(new TopicPartition("test", 0), MemoryRecords.readableRecords(ByteBuffer.allocate(10)));
return new ProduceRequest((short) 1, 5000, produceData);
}
- private AbstractRequestResponse createProduceResponse() {
+ private ProduceResponse createProduceResponse() {
Map<TopicPartition, ProduceResponse.PartitionResponse> responseData = new HashMap<>();
responseData.put(new TopicPartition("test", 0), new ProduceResponse.PartitionResponse(Errors.NONE.code(), 10000, Record.NO_TIMESTAMP));
return new ProduceResponse(responseData, 0);
}
- private AbstractRequest createStopReplicaRequest(boolean deletePartitions) {
+ private StopReplicaRequest createStopReplicaRequest(boolean deletePartitions) {
Set<TopicPartition> partitions = new HashSet<>(Arrays.asList(new TopicPartition("test", 0)));
return new StopReplicaRequest(0, 1, deletePartitions, partitions);
}
- private AbstractRequestResponse createStopReplicaResponse() {
+ private StopReplicaResponse createStopReplicaResponse() {
Map<TopicPartition, Short> responses = new HashMap<>();
responses.put(new TopicPartition("test", 0), Errors.NONE.code());
return new StopReplicaResponse(Errors.NONE.code(), responses);
}
- private AbstractRequest createControlledShutdownRequest() {
+ private ControlledShutdownRequest createControlledShutdownRequest() {
return new ControlledShutdownRequest(10);
}
@@ -423,7 +459,7 @@ public class RequestResponseTest {
return new ControlledShutdownResponse(Errors.NONE.code(), topicPartitions);
}
- private AbstractRequest createLeaderAndIsrRequest() {
+ private LeaderAndIsrRequest createLeaderAndIsrRequest() {
Map<TopicPartition, PartitionState> partitionStates = new HashMap<>();
List<Integer> isr = Arrays.asList(1, 2);
List<Integer> replicas = Arrays.asList(1, 2, 3, 4);
@@ -442,14 +478,14 @@ public class RequestResponseTest {
return new LeaderAndIsrRequest(1, 10, partitionStates, leaders);
}
- private AbstractRequestResponse createLeaderAndIsrResponse() {
+ private LeaderAndIsrResponse createLeaderAndIsrResponse() {
Map<TopicPartition, Short> responses = new HashMap<>();
responses.put(new TopicPartition("test", 0), Errors.NONE.code());
return new LeaderAndIsrResponse(Errors.NONE.code(), responses);
}
@SuppressWarnings("deprecation")
- private AbstractRequest createUpdateMetadataRequest(int version, String rack) {
+ private UpdateMetadataRequest createUpdateMetadataRequest(int version, String rack) {
Map<TopicPartition, PartitionState> partitionStates = new HashMap<>();
List<Integer> isr = Arrays.asList(1, 2);
List<Integer> replicas = Arrays.asList(1, 2, 3, 4);
@@ -482,28 +518,28 @@ public class RequestResponseTest {
}
}
- private AbstractRequestResponse createUpdateMetadataResponse() {
+ private UpdateMetadataResponse createUpdateMetadataResponse() {
return new UpdateMetadataResponse(Errors.NONE.code());
}
- private AbstractRequest createSaslHandshakeRequest() {
+ private SaslHandshakeRequest createSaslHandshakeRequest() {
return new SaslHandshakeRequest("PLAIN");
}
- private AbstractRequestResponse createSaslHandshakeResponse() {
+ private SaslHandshakeResponse createSaslHandshakeResponse() {
return new SaslHandshakeResponse(Errors.NONE.code(), Collections.singletonList("GSSAPI"));
}
- private AbstractRequest createApiVersionRequest() {
+ private ApiVersionsRequest createApiVersionRequest() {
return new ApiVersionsRequest();
}
- private AbstractRequestResponse createApiVersionResponse() {
+ private ApiVersionsResponse createApiVersionResponse() {
List<ApiVersionsResponse.ApiVersion> apiVersions = Arrays.asList(new ApiVersionsResponse.ApiVersion((short) 0, (short) 0, (short) 2));
return new ApiVersionsResponse(Errors.NONE.code(), apiVersions);
}
- private AbstractRequest createCreateTopicRequest() {
+ private CreateTopicsRequest createCreateTopicRequest() {
CreateTopicsRequest.TopicDetails request1 = new CreateTopicsRequest.TopicDetails(3, (short) 5);
Map<Integer, List<Integer>> replicaAssignments = new HashMap<>();
@@ -521,21 +557,65 @@ public class RequestResponseTest {
return new CreateTopicsRequest(request, 0);
}
- private AbstractRequestResponse createCreateTopicResponse() {
+ private CreateTopicsResponse createCreateTopicResponse() {
Map<String, Errors> errors = new HashMap<>();
errors.put("t1", Errors.INVALID_TOPIC_EXCEPTION);
errors.put("t2", Errors.LEADER_NOT_AVAILABLE);
return new CreateTopicsResponse(errors);
}
- private AbstractRequest createDeleteTopicsRequest() {
+ private DeleteTopicsRequest createDeleteTopicsRequest() {
return new DeleteTopicsRequest(new HashSet<>(Arrays.asList("my_t1", "my_t2")), 10000);
}
- private AbstractRequestResponse createDeleteTopicsResponse() {
+ private DeleteTopicsResponse createDeleteTopicsResponse() {
Map<String, Errors> errors = new HashMap<>();
errors.put("t1", Errors.INVALID_TOPIC_EXCEPTION);
errors.put("t2", Errors.TOPIC_AUTHORIZATION_FAILED);
return new DeleteTopicsResponse(errors);
}
+
+ private static class ByteBufferChannel implements GatheringByteChannel {
+ private final ByteBuffer buf;
+ private boolean closed = false;
+
+ private ByteBufferChannel(long size) {
+ this.buf = ByteBuffer.allocate(Long.valueOf(size).intValue());
+ }
+
+ @Override
+ public long write(ByteBuffer[] srcs, int offset, int length) throws IOException {
+ int position = buf.position();
+ for (int i = 0; i < length; i++) {
+ ByteBuffer src = srcs[i].duplicate();
+ if (i == 0)
+ src.position(offset);
+ buf.put(src);
+ }
+ return buf.position() - position;
+ }
+
+ @Override
+ public long write(ByteBuffer[] srcs) throws IOException {
+ return write(srcs, 0, srcs.length);
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ int position = buf.position();
+ buf.put(src);
+ return buf.position() - position;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return !closed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ buf.flip();
+ closed = true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
index 97fe3d8..27c5695 100644
--- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java
@@ -12,18 +12,6 @@
*/
package org.apache.kafka.common.security.authenticator;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
import org.apache.kafka.clients.NetworkClient;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.config.SaslConfigs;
@@ -36,17 +24,17 @@ import org.apache.kafka.common.network.NetworkSend;
import org.apache.kafka.common.network.NetworkTestUtils;
import org.apache.kafka.common.network.NioEchoServer;
import org.apache.kafka.common.network.Selector;
+import org.apache.kafka.common.network.Send;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.Protocol;
import org.apache.kafka.common.protocol.SecurityProtocol;
-import org.apache.kafka.common.protocol.types.Struct;
-import org.apache.kafka.common.requests.AbstractRequestResponse;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.ApiVersionsRequest;
import org.apache.kafka.common.requests.ApiVersionsResponse;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.RequestHeader;
-import org.apache.kafka.common.requests.RequestSend;
import org.apache.kafka.common.requests.ResponseHeader;
import org.apache.kafka.common.requests.SaslHandshakeRequest;
import org.apache.kafka.common.requests.SaslHandshakeResponse;
@@ -55,6 +43,18 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
/**
* Tests for the Sasl authenticator. These use a test harness that runs a simple socket server that echos back responses.
*/
@@ -261,7 +261,8 @@ public class SaslAuthenticatorTest {
String node = "1";
createClientConnection(SecurityProtocol.PLAINTEXT, node);
RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS.id, Short.MAX_VALUE, "someclient", 1);
- selector.send(new NetworkSend(node, RequestSend.serialize(header, new ApiVersionsRequest().toStruct())));
+ ApiVersionsRequest request = new ApiVersionsRequest();
+ selector.send(request.toSend(node, header));
ByteBuffer responseBuffer = waitForResponse();
ResponseHeader.parse(responseBuffer);
ApiVersionsResponse response = ApiVersionsResponse.parse(responseBuffer);
@@ -291,8 +292,9 @@ public class SaslAuthenticatorTest {
// Send ApiVersionsRequest and validate error response.
String node1 = "invalid1";
createClientConnection(SecurityProtocol.PLAINTEXT, node1);
+ SaslHandshakeRequest request = new SaslHandshakeRequest("PLAIN");
RequestHeader header = new RequestHeader(ApiKeys.SASL_HANDSHAKE.id, Short.MAX_VALUE, "someclient", 2);
- selector.send(new NetworkSend(node1, RequestSend.serialize(header, new SaslHandshakeRequest("PLAIN").toStruct())));
+ selector.send(request.toSend(node1, header));
NetworkTestUtils.waitForChannelClose(selector, node1);
selector.close();
@@ -355,8 +357,9 @@ public class SaslAuthenticatorTest {
createClientConnection(SecurityProtocol.PLAINTEXT, node1);
sendHandshakeRequestReceiveResponse(node1);
+ ApiVersionsRequest request = new ApiVersionsRequest();
RequestHeader versionsHeader = new RequestHeader(ApiKeys.API_VERSIONS.id, "someclient", 2);
- selector.send(new NetworkSend(node1, RequestSend.serialize(versionsHeader, new ApiVersionsRequest().toStruct())));
+ selector.send(request.toSend(node1, versionsHeader));
NetworkTestUtils.waitForChannelClose(selector, node1);
selector.close();
@@ -420,7 +423,7 @@ public class SaslAuthenticatorTest {
createClientConnection(SecurityProtocol.PLAINTEXT, node1);
RequestHeader metadataRequestHeader1 = new RequestHeader(ApiKeys.METADATA.id, "someclient", 1);
MetadataRequest metadataRequest1 = new MetadataRequest(Collections.singletonList("sometopic"));
- selector.send(new NetworkSend(node1, RequestSend.serialize(metadataRequestHeader1, metadataRequest1.toStruct())));
+ selector.send(metadataRequest1.toSend(node1, metadataRequestHeader1));
NetworkTestUtils.waitForChannelClose(selector, node1);
selector.close();
@@ -433,7 +436,7 @@ public class SaslAuthenticatorTest {
sendHandshakeRequestReceiveResponse(node2);
RequestHeader metadataRequestHeader2 = new RequestHeader(ApiKeys.METADATA.id, "someclient", 2);
MetadataRequest metadataRequest2 = new MetadataRequest(Collections.singletonList("sometopic"));
- selector.send(new NetworkSend(node2, RequestSend.serialize(metadataRequestHeader2, metadataRequest2.toStruct())));
+ selector.send(metadataRequest2.toSend(node2, metadataRequestHeader2));
NetworkTestUtils.waitForChannelClose(selector, node2);
selector.close();
@@ -581,25 +584,24 @@ public class SaslAuthenticatorTest {
selector = null;
}
- private Struct sendKafkaRequestReceiveResponse(String node, ApiKeys apiKey, AbstractRequestResponse request) throws IOException {
+ private AbstractResponse sendKafkaRequestReceiveResponse(String node, ApiKeys apiKey, AbstractRequest request) throws IOException {
RequestHeader header = new RequestHeader(apiKey.id, "someclient", 1);
- selector.send(new NetworkSend(node, RequestSend.serialize(header, request.toStruct())));
+ Send send = request.toSend(node, header);
+ selector.send(send);
ByteBuffer responseBuffer = waitForResponse();
return NetworkClient.parseResponse(responseBuffer, header);
}
private SaslHandshakeResponse sendHandshakeRequestReceiveResponse(String node) throws Exception {
SaslHandshakeRequest handshakeRequest = new SaslHandshakeRequest("PLAIN");
- Struct responseStruct = sendKafkaRequestReceiveResponse(node, ApiKeys.SASL_HANDSHAKE, handshakeRequest);
- SaslHandshakeResponse response = new SaslHandshakeResponse(responseStruct);
+ SaslHandshakeResponse response = (SaslHandshakeResponse) sendKafkaRequestReceiveResponse(node, ApiKeys.SASL_HANDSHAKE, handshakeRequest);
assertEquals(Errors.NONE.code(), response.errorCode());
return response;
}
private ApiVersionsResponse sendVersionRequestReceiveResponse(String node) throws Exception {
ApiVersionsRequest handshakeRequest = new ApiVersionsRequest();
- Struct responseStruct = sendKafkaRequestReceiveResponse(node, ApiKeys.API_VERSIONS, handshakeRequest);
- ApiVersionsResponse response = new ApiVersionsResponse(responseStruct);
+ ApiVersionsResponse response = (ApiVersionsResponse) sendKafkaRequestReceiveResponse(node, ApiKeys.API_VERSIONS, handshakeRequest);
assertEquals(Errors.NONE.code(), response.errorCode());
return response;
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
index 3bfa83f..3aff8f2 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorTest.java
@@ -16,7 +16,6 @@
**/
package org.apache.kafka.connect.runtime.distributed;
-import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient;
@@ -24,7 +23,7 @@ import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.protocol.types.Struct;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.GroupCoordinatorResponse;
import org.apache.kafka.common.requests.JoinGroupRequest.ProtocolMetadata;
import org.apache.kafka.common.requests.JoinGroupResponse;
@@ -172,7 +171,7 @@ public class WorkerCoordinatorTest {
final String consumerId = "leader";
- client.prepareResponse(groupMetadataResponse(node, Errors.NONE.code()));
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE.code()));
coordinator.ensureCoordinatorReady();
// normal join group
@@ -182,8 +181,8 @@ public class WorkerCoordinatorTest {
client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberConfigOffsets, Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
return sync.memberId().equals(consumerId) &&
sync.generationId() == 1 &&
sync.groupAssignment().containsKey(consumerId);
@@ -212,15 +211,15 @@ public class WorkerCoordinatorTest {
final String memberId = "member";
- client.prepareResponse(groupMetadataResponse(node, Errors.NONE.code()));
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE.code()));
coordinator.ensureCoordinatorReady();
// normal join group
client.prepareResponse(joinGroupFollowerResponse(1, memberId, "leader", Errors.NONE.code()));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
return sync.memberId().equals(memberId) &&
sync.generationId() == 1 &&
sync.groupAssignment().isEmpty();
@@ -253,15 +252,15 @@ public class WorkerCoordinatorTest {
final String memberId = "member";
- client.prepareResponse(groupMetadataResponse(node, Errors.NONE.code()));
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE.code()));
coordinator.ensureCoordinatorReady();
// config mismatch results in assignment error
client.prepareResponse(joinGroupFollowerResponse(1, memberId, "leader", Errors.NONE.code()));
MockClient.RequestMatcher matcher = new MockClient.RequestMatcher() {
@Override
- public boolean matches(ClientRequest request) {
- SyncGroupRequest sync = new SyncGroupRequest(request.request().body());
+ public boolean matches(AbstractRequest body) {
+ SyncGroupRequest sync = (SyncGroupRequest) body;
return sync.memberId().equals(memberId) &&
sync.generationId() == 1 &&
sync.groupAssignment().isEmpty();
@@ -284,7 +283,7 @@ public class WorkerCoordinatorTest {
PowerMock.replayAll();
- client.prepareResponse(groupMetadataResponse(node, Errors.NONE.code()));
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE.code()));
coordinator.ensureCoordinatorReady();
// join the group once
@@ -392,12 +391,11 @@ public class WorkerCoordinatorTest {
}
- private Struct groupMetadataResponse(Node node, short error) {
- GroupCoordinatorResponse response = new GroupCoordinatorResponse(error, node);
- return response.toStruct();
+ private GroupCoordinatorResponse groupCoordinatorResponse(Node node, short error) {
+ return new GroupCoordinatorResponse(error, node);
}
- private Struct joinGroupLeaderResponse(int generationId, String memberId,
+ private JoinGroupResponse joinGroupLeaderResponse(int generationId, String memberId,
Map<String, Long> configOffsets, short error) {
Map<String, ByteBuffer> metadata = new HashMap<>();
for (Map.Entry<String, Long> configStateEntry : configOffsets.entrySet()) {
@@ -407,22 +405,21 @@ public class WorkerCoordinatorTest {
ByteBuffer buf = ConnectProtocol.serializeMetadata(new ConnectProtocol.WorkerState(memberUrl, configOffset));
metadata.put(configStateEntry.getKey(), buf);
}
- return new JoinGroupResponse(error, generationId, WorkerCoordinator.DEFAULT_SUBPROTOCOL, memberId, memberId, metadata).toStruct();
+ return new JoinGroupResponse(error, generationId, WorkerCoordinator.DEFAULT_SUBPROTOCOL, memberId, memberId, metadata);
}
- private Struct joinGroupFollowerResponse(int generationId, String memberId, String leaderId, short error) {
+ private JoinGroupResponse joinGroupFollowerResponse(int generationId, String memberId, String leaderId, short error) {
return new JoinGroupResponse(error, generationId, WorkerCoordinator.DEFAULT_SUBPROTOCOL, memberId, leaderId,
- Collections.<String, ByteBuffer>emptyMap()).toStruct();
+ Collections.<String, ByteBuffer>emptyMap());
}
- private Struct syncGroupResponse(short assignmentError, String leader, long configOffset, List<String> connectorIds,
+ private SyncGroupResponse syncGroupResponse(short assignmentError, String leader, long configOffset, List<String> connectorIds,
List<ConnectorTaskId> taskIds, short error) {
ConnectProtocol.Assignment assignment = new ConnectProtocol.Assignment(assignmentError, leader, LEADER_URL, configOffset, connectorIds, taskIds);
ByteBuffer buf = ConnectProtocol.serializeAssignment(assignment);
- return new SyncGroupResponse(error, buf).toStruct();
+ return new SyncGroupResponse(error, buf);
}
-
private static class MockRebalanceListener implements WorkerRebalanceListener {
public ConnectProtocol.Assignment assignment = null;
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/admin/AdminClient.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/admin/AdminClient.scala b/core/src/main/scala/kafka/admin/AdminClient.scala
index 1179557..592fecf 100644
--- a/core/src/main/scala/kafka/admin/AdminClient.scala
+++ b/core/src/main/scala/kafka/admin/AdminClient.scala
@@ -42,7 +42,7 @@ class AdminClient(val time: Time,
private def send(target: Node,
api: ApiKeys,
- request: AbstractRequest): Struct = {
+ request: AbstractRequest): AbstractResponse = {
var future: RequestFuture[ClientResponse] = null
future = client.send(target, api, request)
@@ -54,9 +54,8 @@ class AdminClient(val time: Time,
throw future.exception()
}
- private def sendAnyNode(api: ApiKeys, request: AbstractRequest): Struct = {
- bootstrapBrokers.foreach {
- case broker =>
+ private def sendAnyNode(api: ApiKeys, request: AbstractRequest): AbstractResponse = {
+ bootstrapBrokers.foreach { broker =>
try {
return send(broker, api, request)
} catch {
@@ -69,23 +68,20 @@ class AdminClient(val time: Time,
private def findCoordinator(groupId: String): Node = {
val request = new GroupCoordinatorRequest(groupId)
- val responseBody = sendAnyNode(ApiKeys.GROUP_COORDINATOR, request)
- val response = new GroupCoordinatorResponse(responseBody)
+ val response = sendAnyNode(ApiKeys.GROUP_COORDINATOR, request).asInstanceOf[GroupCoordinatorResponse]
Errors.forCode(response.errorCode()).maybeThrow()
response.node()
}
def listGroups(node: Node): List[GroupOverview] = {
- val responseBody = send(node, ApiKeys.LIST_GROUPS, new ListGroupsRequest())
- val response = new ListGroupsResponse(responseBody)
+ val response = send(node, ApiKeys.LIST_GROUPS, new ListGroupsRequest()).asInstanceOf[ListGroupsResponse]
Errors.forCode(response.errorCode()).maybeThrow()
response.groups().map(group => GroupOverview(group.groupId(), group.protocolType())).toList
}
private def findAllBrokers(): List[Node] = {
val request = new MetadataRequest(List[String]())
- val responseBody = sendAnyNode(ApiKeys.METADATA, request)
- val response = new MetadataResponse(responseBody)
+ val response = sendAnyNode(ApiKeys.METADATA, request).asInstanceOf[MetadataResponse]
val errors = response.errors()
if (!errors.isEmpty)
debug(s"Metadata request contained errors: $errors")
@@ -140,7 +136,7 @@ class AdminClient(val time: Time,
def describeConsumerGroup(groupId: String): ConsumerGroupSummary = {
val coordinator = findCoordinator(groupId)
val responseBody = send(coordinator, ApiKeys.DESCRIBE_GROUPS, new DescribeGroupsRequest(List(groupId).asJava))
- val response = new DescribeGroupsResponse(responseBody)
+ val response = responseBody.asInstanceOf[DescribeGroupsResponse]
val metadata = response.groups.get(groupId)
if (metadata == null)
throw new KafkaException(s"Response from broker contained no metadata for group $groupId")
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/FetchRequest.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/FetchRequest.scala b/core/src/main/scala/kafka/api/FetchRequest.scala
index 57e99c1..00897db 100644
--- a/core/src/main/scala/kafka/api/FetchRequest.scala
+++ b/core/src/main/scala/kafka/api/FetchRequest.scala
@@ -199,8 +199,7 @@ case class FetchRequest(versionId: Short = FetchRequest.CurrentVersion,
val fetchResponsePartitionData = requestInfo.map { case (topicAndPartition, _) =>
(topicAndPartition, FetchResponsePartitionData(Errors.forException(e).code, -1, MessageSet.Empty))
}
- val fetchRequest = request.requestObj.asInstanceOf[FetchRequest]
- val errorResponse = FetchResponse(correlationId, fetchResponsePartitionData, fetchRequest.versionId)
+ val errorResponse = FetchResponse(correlationId, fetchResponsePartitionData, request.header.apiVersion)
// Magic value does not matter here because the message set is empty
requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.connectionId, errorResponse)))
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/FetchResponse.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala
index d99bbcd..d31d4ba 100644
--- a/core/src/main/scala/kafka/api/FetchResponse.scala
+++ b/core/src/main/scala/kafka/api/FetchResponse.scala
@@ -76,7 +76,8 @@ class PartitionDataSend(val partitionId: Int,
written += channel.write(buffer)
if (!buffer.hasRemaining) {
if (messagesSentSize < messageSize) {
- val bytesSent = partitionData.messages.writeTo(channel, messagesSentSize, messageSize - messagesSentSize)
+ val records = partitionData.messages.asRecords
+ val bytesSent = records.writeTo(channel, messagesSentSize, messageSize - messagesSentSize).toInt
messagesSentSize += bytesSent
written += bytesSent
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/GenericRequestAndHeader.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/GenericRequestAndHeader.scala b/core/src/main/scala/kafka/api/GenericRequestAndHeader.scala
index cb5b95e..3783c29 100644
--- a/core/src/main/scala/kafka/api/GenericRequestAndHeader.scala
+++ b/core/src/main/scala/kafka/api/GenericRequestAndHeader.scala
@@ -14,13 +14,14 @@
package kafka.api
import java.nio.ByteBuffer
-import org.apache.kafka.common.requests.AbstractRequestResponse
+
import kafka.api.ApiUtils._
+import org.apache.kafka.common.requests.AbstractResponse
private[kafka] abstract class GenericRequestAndHeader(val versionId: Short,
val correlationId: Int,
val clientId: String,
- val body: AbstractRequestResponse,
+ val body: AbstractResponse,
val name: String,
override val requestId: Option[Short] = None)
extends RequestOrResponse(requestId) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/GenericResponseAndHeader.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/GenericResponseAndHeader.scala b/core/src/main/scala/kafka/api/GenericResponseAndHeader.scala
index 2835fb6..be0c080 100644
--- a/core/src/main/scala/kafka/api/GenericResponseAndHeader.scala
+++ b/core/src/main/scala/kafka/api/GenericResponseAndHeader.scala
@@ -14,10 +14,11 @@
package kafka.api
import java.nio.ByteBuffer
-import org.apache.kafka.common.requests.AbstractRequestResponse
+
+import org.apache.kafka.common.requests.AbstractResponse
private[kafka] abstract class GenericResponseAndHeader(val correlationId: Int,
- val body: AbstractRequestResponse,
+ val body: AbstractResponse,
val name: String,
override val requestId: Option[Short] = None)
extends RequestOrResponse(requestId) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/ProducerRequest.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/ProducerRequest.scala b/core/src/main/scala/kafka/api/ProducerRequest.scala
index 3ca7bd7..f6e4475 100644
--- a/core/src/main/scala/kafka/api/ProducerRequest.scala
+++ b/core/src/main/scala/kafka/api/ProducerRequest.scala
@@ -128,8 +128,8 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion,
describe(true)
}
- override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = {
- if(request.requestObj.asInstanceOf[ProducerRequest].requiredAcks == 0) {
+ override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = {
+ if(request.body.asInstanceOf[org.apache.kafka.common.requests.ProduceRequest].acks == 0) {
requestChannel.closeConnection(request.processor, request)
}
else {
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/api/RequestOrResponse.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/api/RequestOrResponse.scala b/core/src/main/scala/kafka/api/RequestOrResponse.scala
index 65b37fd..d013047 100644
--- a/core/src/main/scala/kafka/api/RequestOrResponse.scala
+++ b/core/src/main/scala/kafka/api/RequestOrResponse.scala
@@ -42,6 +42,6 @@ abstract class RequestOrResponse(val requestId: Option[Short] = None) extends Lo
* This API has no meaning for a Response object.
* @param details If this is false, omit the parts of the request description that are proportional to the number of
* topics or partitions. This is mainly to control the amount of request logging. */
- def describe(details: Boolean):String
+ def describe(details: Boolean): String
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
index 03cd98c..1935ea2 100755
--- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
+++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala
@@ -42,7 +42,7 @@ class ControllerChannelManager(controllerContext: ControllerContext, config: Kaf
private val brokerLock = new Object
this.logIdent = "[Channel manager on controller " + config.brokerId + "]: "
- controllerContext.liveBrokers.foreach(addNewBroker(_))
+ controllerContext.liveBrokers.foreach(addNewBroker)
def startup() = {
brokerLock synchronized {
@@ -56,7 +56,7 @@ class ControllerChannelManager(controllerContext: ControllerContext, config: Kaf
}
}
- def sendRequest(brokerId: Int, apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractRequestResponse => Unit = null) {
+ def sendRequest(brokerId: Int, apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractResponse => Unit = null) {
brokerLock synchronized {
val stateInfoOpt = brokerStateInfo.get(brokerId)
stateInfoOpt match {
@@ -149,7 +149,7 @@ class ControllerChannelManager(controllerContext: ControllerContext, config: Kaf
}
}
-case class QueueItem(apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractRequestResponse => Unit)
+case class QueueItem(apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractResponse => Unit)
class RequestSendThread(val controllerId: Int,
val controllerContext: ControllerContext,
@@ -185,28 +185,27 @@ class RequestSendThread(val controllerId: Int,
}
else {
val requestHeader = apiVersion.fold(networkClient.nextRequestHeader(apiKey))(networkClient.nextRequestHeader(apiKey, _))
- val send = new RequestSend(brokerNode.idString, requestHeader, request.toStruct)
- val clientRequest = new ClientRequest(time.milliseconds(), true, send, null)
- clientResponse = networkClient.blockingSendAndReceive(clientRequest)(time)
+ val clientRequest = new ClientRequest(brokerNode.idString, time.milliseconds(), true, requestHeader, request, null)
+ clientResponse = networkClient.blockingSendAndReceive(clientRequest, request)(time)
isSendSuccessful = true
}
} catch {
case e: Throwable => // if the send was not successful, reconnect to broker and resend the message
warn(("Controller %d epoch %d fails to send request %s to broker %s. " +
"Reconnecting to broker.").format(controllerId, controllerContext.epoch,
- request.toString, brokerNode.toString()), e)
+ request.toString, brokerNode.toString), e)
networkClient.close(brokerNode.idString)
isSendSuccessful = false
backoff()
}
}
if (clientResponse != null) {
- val response = ApiKeys.forId(clientResponse.request.request.header.apiKey) match {
- case ApiKeys.LEADER_AND_ISR => new LeaderAndIsrResponse(clientResponse.responseBody)
- case ApiKeys.STOP_REPLICA => new StopReplicaResponse(clientResponse.responseBody)
- case ApiKeys.UPDATE_METADATA_KEY => new UpdateMetadataResponse(clientResponse.responseBody)
- case apiKey => throw new KafkaException(s"Unexpected apiKey received: $apiKey")
- }
+ val api = ApiKeys.forId(clientResponse.requestHeader.apiKey)
+ if (api != ApiKeys.LEADER_AND_ISR && api != ApiKeys.STOP_REPLICA && api != ApiKeys.UPDATE_METADATA_KEY)
+ throw new KafkaException(s"Unexpected apiKey received: $apiKey")
+
+ val response = clientResponse.responseBody
+
stateChangeLogger.trace("Controller %d epoch %d received response %s for a request sent to broker %s"
.format(controllerId, controllerContext.epoch, response.toString, brokerNode.toString))
@@ -217,7 +216,7 @@ class RequestSendThread(val controllerId: Int,
}
} catch {
case e: Throwable =>
- error("Controller %d fails to send a request to broker %s".format(controllerId, brokerNode.toString()), e)
+ error("Controller %d fails to send a request to broker %s".format(controllerId, brokerNode.toString), e)
// If there is any socket error (eg, socket timeout), the connection is no longer usable and needs to be recreated.
networkClient.close(brokerNode.idString)
}
@@ -230,13 +229,13 @@ class RequestSendThread(val controllerId: Int,
if (!networkClient.blockingReady(brokerNode, socketTimeoutMs)(time))
throw new SocketTimeoutException(s"Failed to connect within $socketTimeoutMs ms")
- info("Controller %d connected to %s for sending state change requests".format(controllerId, brokerNode.toString()))
+ info("Controller %d connected to %s for sending state change requests".format(controllerId, brokerNode.toString))
}
true
} catch {
case e: Throwable =>
- warn("Controller %d's connection to broker %s was unsuccessful".format(controllerId, brokerNode.toString()), e)
+ warn("Controller %d's connection to broker %s was unsuccessful".format(controllerId, brokerNode.toString), e)
networkClient.close(brokerNode.idString)
false
}
@@ -273,7 +272,7 @@ class ControllerBrokerRequestBatch(controller: KafkaController) extends Logging
def addLeaderAndIsrRequestForBrokers(brokerIds: Seq[Int], topic: String, partition: Int,
leaderIsrAndControllerEpoch: LeaderIsrAndControllerEpoch,
- replicas: Seq[Int], callback: AbstractRequestResponse => Unit = null) {
+ replicas: Seq[Int], callback: AbstractResponse => Unit = null) {
val topicPartition = new TopicPartition(topic, partition)
brokerIds.filter(_ >= 0).foreach { brokerId =>
@@ -286,13 +285,13 @@ class ControllerBrokerRequestBatch(controller: KafkaController) extends Logging
}
def addStopReplicaRequestForBrokers(brokerIds: Seq[Int], topic: String, partition: Int, deletePartition: Boolean,
- callback: (AbstractRequestResponse, Int) => Unit = null) {
+ callback: (AbstractResponse, Int) => Unit = null) {
brokerIds.filter(b => b >= 0).foreach { brokerId =>
stopReplicaRequestMap.getOrElseUpdate(brokerId, Seq.empty[StopReplicaRequestInfo])
val v = stopReplicaRequestMap(brokerId)
if(callback != null)
stopReplicaRequestMap(brokerId) = v :+ StopReplicaRequestInfo(PartitionAndReplica(topic, partition, brokerId),
- deletePartition, (r: AbstractRequestResponse) => callback(r, brokerId))
+ deletePartition, (r: AbstractResponse) => callback(r, brokerId))
else
stopReplicaRequestMap(brokerId) = v :+ StopReplicaRequestInfo(PartitionAndReplica(topic, partition, brokerId),
deletePartition)
@@ -302,7 +301,7 @@ class ControllerBrokerRequestBatch(controller: KafkaController) extends Logging
/** Send UpdateMetadataRequest to the given brokers for the given partitions and partitions that are being deleted */
def addUpdateMetadataRequestForBrokers(brokerIds: Seq[Int],
partitions: collection.Set[TopicAndPartition] = Set.empty[TopicAndPartition],
- callback: AbstractRequestResponse => Unit = null) {
+ callback: AbstractResponse => Unit = null) {
def updateMetadataRequestMapFor(partition: TopicAndPartition, beingDeleted: Boolean) {
val leaderIsrAndControllerEpochOpt = controllerContext.partitionLeadershipInfo.get(partition)
leaderIsrAndControllerEpochOpt match {
@@ -444,29 +443,29 @@ case class ControllerBrokerStateInfo(networkClient: NetworkClient,
messageQueue: BlockingQueue[QueueItem],
requestSendThread: RequestSendThread)
-case class StopReplicaRequestInfo(replica: PartitionAndReplica, deletePartition: Boolean, callback: AbstractRequestResponse => Unit = null)
+case class StopReplicaRequestInfo(replica: PartitionAndReplica, deletePartition: Boolean, callback: AbstractResponse => Unit = null)
-class Callbacks private (var leaderAndIsrResponseCallback: AbstractRequestResponse => Unit = null,
- var updateMetadataResponseCallback: AbstractRequestResponse => Unit = null,
- var stopReplicaResponseCallback: (AbstractRequestResponse, Int) => Unit = null)
+class Callbacks private (var leaderAndIsrResponseCallback: AbstractResponse => Unit = null,
+ var updateMetadataResponseCallback: AbstractResponse => Unit = null,
+ var stopReplicaResponseCallback: (AbstractResponse, Int) => Unit = null)
object Callbacks {
class CallbackBuilder {
- var leaderAndIsrResponseCbk: AbstractRequestResponse => Unit = null
- var updateMetadataResponseCbk: AbstractRequestResponse => Unit = null
- var stopReplicaResponseCbk: (AbstractRequestResponse, Int) => Unit = null
+ var leaderAndIsrResponseCbk: AbstractResponse => Unit = null
+ var updateMetadataResponseCbk: AbstractResponse => Unit = null
+ var stopReplicaResponseCbk: (AbstractResponse, Int) => Unit = null
- def leaderAndIsrCallback(cbk: AbstractRequestResponse => Unit): CallbackBuilder = {
+ def leaderAndIsrCallback(cbk: AbstractResponse => Unit): CallbackBuilder = {
leaderAndIsrResponseCbk = cbk
this
}
- def updateMetadataCallback(cbk: AbstractRequestResponse => Unit): CallbackBuilder = {
+ def updateMetadataCallback(cbk: AbstractResponse => Unit): CallbackBuilder = {
updateMetadataResponseCbk = cbk
this
}
- def stopReplicaCallback(cbk: (AbstractRequestResponse, Int) => Unit): CallbackBuilder = {
+ def stopReplicaCallback(cbk: (AbstractResponse, Int) => Unit): CallbackBuilder = {
stopReplicaResponseCbk = cbk
this
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/controller/KafkaController.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala
index 4a94aad..730f07c 100755
--- a/core/src/main/scala/kafka/controller/KafkaController.scala
+++ b/core/src/main/scala/kafka/controller/KafkaController.scala
@@ -20,7 +20,7 @@ import java.util
import org.apache.kafka.common.errors.{BrokerNotAvailableException, ControllerMovedException}
import org.apache.kafka.common.protocol.ApiKeys
-import org.apache.kafka.common.requests.{AbstractRequest, AbstractRequestResponse}
+import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse}
import scala.collection._
import com.yammer.metrics.core.{Gauge, Meter}
@@ -39,7 +39,7 @@ import kafka.utils.CoreUtils._
import org.apache.zookeeper.Watcher.Event.KeeperState
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.utils.Time
-import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection}
+import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient}
import org.I0Itec.zkclient.exception.{ZkNoNodeException, ZkNodeExistsException}
import java.util.concurrent.locks.ReentrantLock
@@ -136,7 +136,7 @@ object KafkaController extends Logging {
Json.parseFull(controllerInfoString) match {
case Some(m) =>
val controllerInfo = m.asInstanceOf[Map[String, Any]]
- controllerInfo.get("brokerid").get.asInstanceOf[Int]
+ controllerInfo("brokerid").asInstanceOf[Int]
case None => throw new KafkaException("Failed to parse the controller info json [%s].".format(controllerInfoString))
}
} catch {
@@ -686,7 +686,7 @@ class KafkaController(val config : KafkaConfig, zkUtils: ZkUtils, val brokerStat
onControllerResignation()
}
- def sendRequest(brokerId: Int, apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractRequestResponse => Unit = null) = {
+ def sendRequest(brokerId: Int, apiKey: ApiKeys, apiVersion: Option[Short], request: AbstractRequest, callback: AbstractResponse => Unit = null) = {
controllerContext.controllerChannelManager.sendRequest(brokerId, apiKey, apiVersion, request, callback)
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
index 8e5f3a1..823f9b4 100755
--- a/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
+++ b/core/src/main/scala/kafka/controller/TopicDeletionManager.scala
@@ -19,13 +19,14 @@ package kafka.controller
import kafka.server.ConfigType
import org.apache.kafka.common.protocol.Errors
-import org.apache.kafka.common.requests.{StopReplicaResponse, AbstractRequestResponse}
+import org.apache.kafka.common.requests.{AbstractResponse, StopReplicaResponse}
import collection.mutable
import collection.JavaConverters._
-import kafka.utils.{ShutdownableThread, Logging}
+import kafka.utils.{Logging, ShutdownableThread}
import kafka.utils.CoreUtils._
import kafka.utils.ZkUtils._
+
import collection.Set
import kafka.common.TopicAndPartition
import java.util.concurrent.locks.ReentrantLock
@@ -378,7 +379,7 @@ class TopicDeletionManager(controller: KafkaController,
startReplicaDeletion(replicasPerPartition)
}
- private def deleteTopicStopReplicaCallback(stopReplicaResponseObj: AbstractRequestResponse, replicaId: Int) {
+ private def deleteTopicStopReplicaCallback(stopReplicaResponseObj: AbstractResponse, replicaId: Int) {
val stopReplicaResponse = stopReplicaResponseObj.asInstanceOf[StopReplicaResponse]
debug("Delete topic callback invoked for %s".format(stopReplicaResponse))
val responseMap = stopReplicaResponse.responses.asScala
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/log/FileMessageSet.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/log/FileMessageSet.scala b/core/src/main/scala/kafka/log/FileMessageSet.scala
index fd9ec5f..c33e376 100755
--- a/core/src/main/scala/kafka/log/FileMessageSet.scala
+++ b/core/src/main/scala/kafka/log/FileMessageSet.scala
@@ -26,9 +26,10 @@ import kafka.utils._
import kafka.message._
import kafka.common.KafkaException
import java.util.concurrent.TimeUnit
-import kafka.metrics.{KafkaTimer, KafkaMetricsGroup}
+
+import kafka.metrics.{KafkaMetricsGroup, KafkaTimer}
import org.apache.kafka.common.errors.CorruptRecordException
-import org.apache.kafka.common.network.TransportLayer
+import org.apache.kafka.common.record.FileRecords
import org.apache.kafka.common.utils.Utils
import scala.collection.mutable.ArrayBuffer
@@ -48,7 +49,6 @@ class FileMessageSet private[kafka](@volatile var file: File,
private[log] val start: Int,
private[log] val end: Int,
isSlice: Boolean) extends MessageSet {
- import FileMessageSet._
/* the size of the message set in bytes */
private val _size =
if(isSlice)
@@ -126,6 +126,8 @@ class FileMessageSet private[kafka](@volatile var file: File,
})
}
+ override def asRecords: FileRecords = new FileRecords(file, channel, start, end, isSlice)
+
/**
* Search forward for the file position of the last offset that is greater than or equal to the target offset
* and return its physical position and the size of the message (including log overhead) at the returned offset. If
@@ -206,31 +208,6 @@ class FileMessageSet private[kafka](@volatile var file: File,
}
/**
- * Write some of this set to the given channel.
- * @param destChannel The channel to write to.
- * @param writePosition The position in the message set to begin writing from.
- * @param size The maximum number of bytes to write
- * @return The number of bytes actually written.
- */
- def writeTo(destChannel: GatheringByteChannel, writePosition: Long, size: Int): Int = {
- // Ensure that the underlying size has not changed.
- val newSize = math.min(channel.size.toInt, end) - start
- if (newSize < _size.get()) {
- throw new KafkaException("Size of FileMessageSet %s has been truncated during write: old size %d, new size %d"
- .format(file.getAbsolutePath, _size.get(), newSize))
- }
- val position = start + writePosition
- val count = math.min(size, sizeInBytes)
- val bytesTransferred = (destChannel match {
- case tl: TransportLayer => tl.transferFrom(channel, position, count)
- case dc => channel.transferTo(position, count, dc)
- }).toInt
- trace("FileMessageSet " + file.getAbsolutePath + " : bytes transferred : " + bytesTransferred
- + " bytes requested for transfer : " + math.min(size, sizeInBytes))
- bytesTransferred
- }
-
- /**
* This method is called before we write messages to the socket using zero-copy transfer. We need to
* make sure all the messages in the message set have the expected magic value.
*
@@ -337,7 +314,7 @@ class FileMessageSet private[kafka](@volatile var file: File,
// increment the location and return the item
location += size + sizeOffsetLength
- new MessageAndOffset(new Message(buffer), offset)
+ MessageAndOffset(new Message(buffer), offset)
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/message/ByteBufferMessageSet.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/message/ByteBufferMessageSet.scala b/core/src/main/scala/kafka/message/ByteBufferMessageSet.scala
index a33bc4b..096344d 100644
--- a/core/src/main/scala/kafka/message/ByteBufferMessageSet.scala
+++ b/core/src/main/scala/kafka/message/ByteBufferMessageSet.scala
@@ -26,7 +26,7 @@ import java.util.ArrayDeque
import kafka.message.ByteBufferMessageSet.FilterResult
import org.apache.kafka.common.errors.InvalidTimestampException
-import org.apache.kafka.common.record.TimestampType
+import org.apache.kafka.common.record.{MemoryRecords, TimestampType}
import org.apache.kafka.common.utils.Utils
import scala.collection.mutable
@@ -352,6 +352,8 @@ class ByteBufferMessageSet(val buffer: ByteBuffer) extends MessageSet with Loggi
def getBuffer = buffer
+ override def asRecords: MemoryRecords = MemoryRecords.readableRecords(buffer.duplicate())
+
private def shallowValidBytes: Int = {
if (shallowValidByteCount < 0) {
this.shallowValidByteCount = this.internalIterator(isShallow = true).map { messageAndOffset =>
@@ -371,19 +373,6 @@ class ByteBufferMessageSet(val buffer: ByteBuffer) extends MessageSet with Loggi
written
}
- /** Write the messages in this set to the given channel starting at the given offset byte.
- * Less than the complete amount may be written, but no more than maxSize can be. The number
- * of bytes written is returned */
- def writeTo(channel: GatheringByteChannel, offset: Long, maxSize: Int): Int = {
- if (offset > Int.MaxValue)
- throw new IllegalArgumentException(s"offset should not be larger than Int.MaxValue: $offset")
- val dup = buffer.duplicate()
- val position = offset.toInt
- dup.position(position)
- dup.limit(math.min(buffer.limit, position + maxSize))
- channel.write(dup)
- }
-
override def isMagicValueInAllWrapperMessages(expectedMagicValue: Byte): Boolean = {
for (messageAndOffset <- shallowIterator) {
if (messageAndOffset.message.magic != expectedMagicValue)
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/message/MessageSet.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/message/MessageSet.scala b/core/src/main/scala/kafka/message/MessageSet.scala
index 14c455c..ffa27fa 100644
--- a/core/src/main/scala/kafka/message/MessageSet.scala
+++ b/core/src/main/scala/kafka/message/MessageSet.scala
@@ -20,6 +20,8 @@ package kafka.message
import java.nio._
import java.nio.channels._
+import org.apache.kafka.common.record.Records
+
/**
* Message set helper functions
*/
@@ -70,11 +72,6 @@ case class MagicAndTimestamp(magic: Byte, timestamp: Long)
*/
abstract class MessageSet extends Iterable[MessageAndOffset] {
- /** Write the messages in this set to the given channel starting at the given offset byte.
- * Less than the complete amount may be written, but no more than maxSize can be. The number
- * of bytes written is returned */
- def writeTo(channel: GatheringByteChannel, offset: Long, maxSize: Int): Int
-
/**
* Check if all the wrapper messages in the message set have the expected magic value
*/
@@ -91,6 +88,11 @@ abstract class MessageSet extends Iterable[MessageAndOffset] {
def sizeInBytes: Int
/**
+ * Get the client representation of the message set
+ */
+ def asRecords: Records
+
+ /**
* Print this message set's contents. If the message set has more than 100 messages, just
* print the first 100.
*/
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/network/RequestChannel.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala
index dace782..0cece68 100644
--- a/core/src/main/scala/kafka/network/RequestChannel.scala
+++ b/core/src/main/scala/kafka/network/RequestChannel.scala
@@ -23,7 +23,7 @@ import java.util.HashMap
import java.util.concurrent._
import com.yammer.metrics.core.Gauge
-import kafka.api._
+import kafka.api.{ControlledShutdownRequest, RequestOrResponse}
import kafka.metrics.KafkaMetricsGroup
import kafka.server.QuotaId
import kafka.utils.{Logging, SystemTime}
@@ -31,19 +31,19 @@ import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.errors.InvalidRequestException
import org.apache.kafka.common.network.Send
import org.apache.kafka.common.protocol.{ApiKeys, Protocol, SecurityProtocol}
-import org.apache.kafka.common.requests.{AbstractRequest, ApiVersionsRequest, ProduceRequest, RequestHeader, RequestSend}
+import org.apache.kafka.common.record.MemoryRecords
+import org.apache.kafka.common.requests._
import org.apache.kafka.common.security.auth.KafkaPrincipal
import org.apache.log4j.Logger
-
object RequestChannel extends Logging {
- val AllDone = new Request(processor = 1, connectionId = "2", new Session(KafkaPrincipal.ANONYMOUS, InetAddress.getLocalHost()), buffer = getShutdownReceive(), startTimeMs = 0, securityProtocol = SecurityProtocol.PLAINTEXT)
+ val AllDone = Request(processor = 1, connectionId = "2", Session(KafkaPrincipal.ANONYMOUS, InetAddress.getLocalHost), buffer = getShutdownReceive(), startTimeMs = 0, securityProtocol = SecurityProtocol.PLAINTEXT)
private val requestLogger = Logger.getLogger("kafka.request.logger")
def getShutdownReceive() = {
val emptyRequestHeader = new RequestHeader(ApiKeys.PRODUCE.id, "", 0)
- val emptyProduceRequest = new ProduceRequest(0, 0, new HashMap[TopicPartition, ByteBuffer]())
- RequestSend.serialize(emptyRequestHeader, emptyProduceRequest.toStruct)
+ val emptyProduceRequest = new ProduceRequest(0, 0, new HashMap[TopicPartition, MemoryRecords]())
+ AbstractRequestResponse.serialize(emptyRequestHeader, emptyProduceRequest)
}
case class Session(principal: KafkaPrincipal, clientAddress: InetAddress) {
@@ -61,19 +61,12 @@ object RequestChannel extends Logging {
val requestId = buffer.getShort()
- // TODO: this will be removed once we migrated to client-side format
- // for server-side request / response format
- // NOTE: this map only includes the server-side request/response handlers. Newer
- // request types should only use the client-side versions which are parsed with
- // o.a.k.common.requests.AbstractRequest.getRequest()
- private val keyToNameAndDeserializerMap: Map[Short, (ByteBuffer) => RequestOrResponse]=
- Map(ApiKeys.FETCH.id -> FetchRequest.readFrom,
- ApiKeys.CONTROLLED_SHUTDOWN_KEY.id -> ControlledShutdownRequest.readFrom
- )
-
- // TODO: this will be removed once we migrated to client-side format
- val requestObj =
- keyToNameAndDeserializerMap.get(requestId).map(readFrom => readFrom(buffer)).orNull
+ // TODO: this will be removed once we remove support for v0 of ControlledShutdownRequest (which
+ // depends on a non-standard request header)
+ val requestObj: RequestOrResponse = if (requestId == ApiKeys.CONTROLLED_SHUTDOWN_KEY.id)
+ ControlledShutdownRequest.readFrom(buffer)
+ else
+ null
// if we failed to find a server-side mapping, then try using the
// client-side request / response format
@@ -108,7 +101,7 @@ object RequestChannel extends Logging {
if (requestObj != null)
requestObj.describe(details)
else
- header.toString + " -- " + body.toString
+ s"$header -- $body"
}
trace("Processor %d received request : %s".format(processor, requestDesc(true)))
@@ -135,7 +128,7 @@ object RequestChannel extends Logging {
val totalTime = endTimeMs - startTimeMs
val fetchMetricNames =
if (requestId == ApiKeys.FETCH.id) {
- val isFromFollower = requestObj.asInstanceOf[FetchRequest].isFromFollower
+ val isFromFollower = body.asInstanceOf[FetchRequest].isFromFollower
Seq(
if (isFromFollower) RequestMetrics.followFetchMetricName
else RequestMetrics.consumerFetchMetricName
@@ -172,6 +165,9 @@ object RequestChannel extends Logging {
def this(request: Request, send: Send) =
this(request.processor, request, send)
+
+ def this(request: Request, response: AbstractResponse) =
+ this(request, response.toSend(request.connectionId, request.header))
}
trait ResponseAction
@@ -221,14 +217,14 @@ class RequestChannel(val numProcessors: Int, val queueSize: Int) extends KafkaMe
/** No operation to take for the request, need to read more over the network */
def noOperation(processor: Int, request: RequestChannel.Request) {
- responseQueues(processor).put(new RequestChannel.Response(processor, request, null, RequestChannel.NoOpAction))
+ responseQueues(processor).put(RequestChannel.Response(processor, request, null, RequestChannel.NoOpAction))
for(onResponse <- responseListeners)
onResponse(processor)
}
/** Close the connection for the request */
def closeConnection(processor: Int, request: RequestChannel.Request) {
- responseQueues(processor).put(new RequestChannel.Response(processor, request, null, RequestChannel.CloseConnectionAction))
+ responseQueues(processor).put(RequestChannel.Response(processor, request, null, RequestChannel.CloseConnectionAction))
for(onResponse <- responseListeners)
onResponse(processor)
}
@@ -254,7 +250,7 @@ class RequestChannel(val numProcessors: Int, val queueSize: Int) extends KafkaMe
}
def shutdown() {
- requestQueue.clear
+ requestQueue.clear()
}
}
@@ -283,4 +279,3 @@ class RequestMetrics(name: String) extends KafkaMetricsGroup {
val responseSendTimeHist = newHistogram("ResponseSendTimeMs", biased = true, tags)
val totalTimeHist = newHistogram("TotalTimeMs", biased = true, tags)
}
-
http://git-wip-us.apache.org/repos/asf/kafka/blob/3b4c3479/core/src/main/scala/kafka/server/DelayedFetch.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/DelayedFetch.scala b/core/src/main/scala/kafka/server/DelayedFetch.scala
index 4bf04e6..001051f 100644
--- a/core/src/main/scala/kafka/server/DelayedFetch.scala
+++ b/core/src/main/scala/kafka/server/DelayedFetch.scala
@@ -20,14 +20,15 @@ package kafka.server
import java.util.concurrent.TimeUnit
import kafka.api.FetchResponsePartitionData
-import kafka.api.PartitionFetchInfo
import kafka.common.TopicAndPartition
import kafka.metrics.KafkaMetricsGroup
+import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.errors.{NotLeaderForPartitionException, UnknownTopicOrPartitionException}
+import org.apache.kafka.common.requests.FetchRequest.PartitionData
import scala.collection._
-case class FetchPartitionStatus(startOffsetMetadata: LogOffsetMetadata, fetchInfo: PartitionFetchInfo) {
+case class FetchPartitionStatus(startOffsetMetadata: LogOffsetMetadata, fetchInfo: PartitionData) {
override def toString = "[startOffsetMetadata: " + startOffsetMetadata + ", " +
"fetchInfo: " + fetchInfo + "]"
@@ -103,7 +104,7 @@ class DelayedFetch(delayMs: Long,
return forceComplete()
} else if (fetchOffset.messageOffset < endOffset.messageOffset) {
// we take the partition fetch size as upper bound when accumulating the bytes (skip if a throttled partition)
- val bytesAvailable = math.min(endOffset.positionDiff(fetchOffset), fetchStatus.fetchInfo.fetchSize)
+ val bytesAvailable = math.min(endOffset.positionDiff(fetchOffset), fetchStatus.fetchInfo.maxBytes)
if (quota.isThrottled(topicAndPartition))
accumulatedThrottledSize += bytesAvailable
else
@@ -146,7 +147,7 @@ class DelayedFetch(delayMs: Long,
readOnlyCommitted = fetchMetadata.fetchOnlyCommitted,
fetchMaxBytes = fetchMetadata.fetchMaxBytes,
hardMaxBytesLimit = fetchMetadata.hardMaxBytesLimit,
- readPartitionInfo = fetchMetadata.fetchPartitionStatus.map { case (tp, status) => tp -> status.fetchInfo },
+ readPartitionInfo = fetchMetadata.fetchPartitionStatus.map { case (tp, status) => new TopicPartition(tp.topic, tp.partition) -> status.fetchInfo },
quota = quota
)