You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2023/10/24 20:03:12 UTC
[kafka] branch trunk updated: KAFKA-14274 [6, 7]: Introduction of fetch request manager (#14406)
This is an automated email from the ASF dual-hosted git repository.
junrao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2b233bfa5f3 KAFKA-14274 [6, 7]: Introduction of fetch request manager (#14406)
2b233bfa5f3 is described below
commit 2b233bfa5f35bf237effe8c5202fd2b80c601943
Author: Kirk True <ki...@kirktrue.pro>
AuthorDate: Tue Oct 24 13:03:05 2023 -0700
KAFKA-14274 [6, 7]: Introduction of fetch request manager (#14406)
Changes:
1. Introduces FetchRequestManager that implements the RequestManager
API for fetching messages from brokers. Unlike Fetcher, record
decompression and deserialization is performed on the application
thread inside CompletedFetch.
2. Restructured the code so that objects owned by the background thread
are not instantiated until the background thread runs (via Supplier)
to ensure that there are no references available to the
application thread.
3. Ensuring resources are properly using Closeable and using
IdempotentCloser to ensure they're only closed once.
4. Introduces ConsumerTestBuilder to reduce a lot of inconsistency in
the way the objects were built up for tests.
Reviewers: Philip Nee <pn...@confluent.io>, Lianet Magrans <li...@gmail.com>, Jun Rao<ju...@gmail.com>
---
checkstyle/import-control.xml | 4 +
checkstyle/suppressions.xml | 12 +-
.../kafka/clients/consumer/KafkaConsumer.java | 11 +-
.../kafka/clients/consumer/MockConsumer.java | 2 +-
.../clients/consumer/internals/AbstractFetch.java | 169 +--
...{ErrorEventHandler.java => CachedSupplier.java} | 29 +-
.../consumer/internals/CommitRequestManager.java | 17 +-
.../clients/consumer/internals/CompletedFetch.java | 28 +-
.../consumer/internals/ConsumerCoordinator.java | 10 +-
.../consumer/internals/ConsumerNetworkThread.java | 268 ++++
.../clients/consumer/internals/ConsumerUtils.java | 18 +-
.../internals/CoordinatorRequestManager.java | 28 +-
.../internals/DefaultBackgroundThread.java | 293 ----
.../consumer/internals/DefaultEventHandler.java | 151 --
.../clients/consumer/internals/Deserializers.java | 8 +
.../clients/consumer/internals/FetchBuffer.java | 169 ++-
.../clients/consumer/internals/FetchCollector.java | 11 +-
.../clients/consumer/internals/FetchConfig.java | 84 +-
.../consumer/internals/FetchRequestManager.java | 117 ++
.../kafka/clients/consumer/internals/Fetcher.java | 130 +-
.../internals/HeartbeatRequestManager.java | 40 +-
.../consumer/internals/NetworkClientDelegate.java | 123 +-
.../consumer/internals/OffsetFetcherUtils.java | 3 +-
.../consumer/internals/OffsetsRequestManager.java | 25 +-
.../consumer/internals/PrototypeAsyncConsumer.java | 759 +++++++---
.../clients/consumer/internals/RequestManager.java | 37 +
.../consumer/internals/RequestManagers.java | 132 +-
.../internals/TopicMetadataRequestManager.java | 6 +-
.../internals/events/ApplicationEvent.java | 2 +-
.../internals/events/ApplicationEventHandler.java | 115 ++
.../events/ApplicationEventProcessor.java | 150 +-
.../consumer/internals/events/BackgroundEvent.java | 6 +-
.../internals/events/BackgroundEventHandler.java | 52 +
.../internals/events/BackgroundEventProcessor.java | 75 +
.../events/CompletableApplicationEvent.java | 2 +-
.../CompletableEvent.java} | 13 +-
.../internals/events/ErrorBackgroundEvent.java | 10 +-
.../consumer/internals/events/EventHandler.java | 65 -
.../consumer/internals/events/EventProcessor.java | 137 ++
.../internals/events/NoopApplicationEvent.java | 62 -
.../internals/events/NoopBackgroundEvent.java | 62 -
.../kafka/clients/consumer/KafkaConsumerTest.java | 4 +-
.../internals/CommitRequestManagerTest.java | 3 +-
.../consumer/internals/CompletedFetchTest.java | 96 +-
.../internals/ConsumerCoordinatorTest.java | 22 +-
.../internals/ConsumerNetworkThreadTest.java | 269 ++++
.../consumer/internals/ConsumerTestBuilder.java | 370 +++++
.../internals/CoordinatorRequestManagerTest.java | 21 +-
.../internals/DefaultBackgroundThreadTest.java | 436 ------
.../internals/DefaultEventHandlerTest.java | 81 --
.../consumer/internals/FetchCollectorTest.java | 10 +-
.../consumer/internals/FetchConfigTest.java | 48 +-
...tcherTest.java => FetchRequestManagerTest.java} | 1491 +++++++++-----------
.../clients/consumer/internals/FetcherTest.java | 546 ++++---
.../internals/HeartbeatRequestManagerTest.java | 201 +--
.../consumer/internals/OffsetFetcherTest.java | 6 +-
.../internals/OffsetsRequestManagerTest.java | 54 +-
.../internals/PrototypeAsyncConsumerTest.java | 208 +--
.../events/BackgroundEventHandlerTest.java | 141 ++
59 files changed, 4146 insertions(+), 3296 deletions(-)
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 46774912418..e8f8bd56547 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -205,6 +205,10 @@
<subpackage name="consumer">
<allow pkg="org.apache.kafka.clients.consumer" />
+
+ <subpackage name="internals">
+ <allow pkg="org.apache.kafka.clients" />
+ </subpackage>
</subpackage>
<subpackage name="producer">
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index bd54231ffe5..e1e746c755a 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -61,6 +61,8 @@
files="AbstractRequest.java"/>
<suppress checks="ClassFanOutComplexity"
files="AbstractResponse.java"/>
+ <suppress checks="ClassFanOutComplexity"
+ files="PrototypeAsyncConsumer.java"/>
<suppress checks="MethodLength"
files="(KerberosLogin|RequestResponseTest|ConnectMetricsRegistry|KafkaConsumer|AbstractStickyAssignor|AbstractRequest|AbstractResponse).java"/>
@@ -68,7 +70,7 @@
<suppress checks="ParameterNumber"
files="(NetworkClient|FieldSpec|KafkaRaftClient).java"/>
<suppress checks="ParameterNumber"
- files="(KafkaConsumer|ConsumerCoordinator).java"/>
+ files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator).java"/>
<suppress checks="ParameterNumber"
files="(RecordAccumulator|Sender).java"/>
<suppress checks="ParameterNumber"
@@ -79,7 +81,7 @@
files="MemoryRecordsBuilder.java"/>
<suppress checks="ClassDataAbstractionCoupling"
- files="(KafkaConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|KafkaRaftClientTest).java"/>
+ files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|KafkaRaftClientTest).java"/>
<suppress checks="ClassDataAbstractionCoupling"
files="(Errors|SaslAuthenticatorTest|AgentTest|CoordinatorTest).java"/>
@@ -108,10 +110,10 @@
<!-- Clients tests -->
<suppress checks="ClassDataAbstractionCoupling"
- files="(Sender|Fetcher|OffsetFetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
+ files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|PrototypeAsyncConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
<suppress checks="ClassFanOutComplexity"
- files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
+ files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
<suppress checks="ClassFanOutComplexity"
files="MockAdminClient.java"/>
@@ -120,7 +122,7 @@
files="(OffsetFetcher|RequestResponse)Test.java"/>
<suppress checks="JavaNCSS"
- files="RequestResponseTest.java|FetcherTest.java|KafkaAdminClientTest.java"/>
+ files="RequestResponseTest.java|FetcherTest.java|FetchRequestManagerTest.java|KafkaAdminClientTest.java"/>
<suppress checks="NPathComplexity"
files="MemoryRecordsTest|MetricsTest|RequestResponseTest|TestSslUtils|AclAuthorizerBenchmark"/>
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index f8221df36ae..dd273c38c43 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -78,14 +78,13 @@ import java.util.regex.Pattern;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createConsumerNetworkClient;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel;
import static org.apache.kafka.common.utils.Utils.closeQuietly;
import static org.apache.kafka.common.utils.Utils.isBlank;
import static org.apache.kafka.common.utils.Utils.join;
@@ -570,7 +569,6 @@ import static org.apache.kafka.common.utils.Utils.swallow;
public class KafkaConsumer<K, V> implements Consumer<K, V> {
private static final long NO_CURRENT_THREAD = -1L;
- static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000;
static final String DEFAULT_REASON = "rebalance enforced by user";
// Visible for testing
@@ -715,7 +713,8 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
this.metadata.bootstrap(addresses);
FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics);
- this.isolationLevel = configuredIsolationLevel(config);
+ FetchConfig fetchConfig = new FetchConfig(config);
+ this.isolationLevel = fetchConfig.isolationLevel;
ApiVersions apiVersions = new ApiVersions();
this.client = createConsumerNetworkClient(config,
@@ -753,13 +752,13 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED),
config.getString(ConsumerConfig.CLIENT_RACK_CONFIG));
}
- FetchConfig<K, V> fetchConfig = createFetchConfig(config, this.deserializers);
this.fetcher = new Fetcher<>(
logContext,
this.client,
this.metadata,
this.subscriptions,
fetchConfig,
+ this.deserializers,
fetchMetricsManager,
this.time);
this.offsetFetcher = new OffsetFetcher(logContext,
@@ -2483,7 +2482,7 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
// coordinator lookup if there are partitions which have missing positions, so
// a consumer with manually assigned partitions can avoid a coordinator dependence
// by always ensuring that assigned partitions have an initial position.
- if (coordinator != null && !coordinator.refreshCommittedOffsetsIfNeeded(timer)) return false;
+ if (coordinator != null && !coordinator.initWithCommittedOffsetsIfNeeded(timer)) return false;
// If there are partitions still needing a position and a reset policy is defined,
// request reset using the default policy. If no reset strategy is defined and there
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
index 26c6dd3e94b..5f143c9a079 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java
@@ -46,7 +46,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static java.util.Collections.singleton;
-import static org.apache.kafka.clients.consumer.KafkaConsumer.DEFAULT_CLOSE_TIMEOUT_MS;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS;
/**
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java
index 4f579bb0e0f..ffca2cee35d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java
@@ -18,10 +18,13 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.FetchSessionHandler;
+import org.apache.kafka.clients.KafkaClient;
+import org.apache.kafka.clients.NetworkClientUtils;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.internals.IdempotentCloser;
import org.apache.kafka.common.message.FetchResponseData;
import org.apache.kafka.common.protocol.ApiKeys;
@@ -37,61 +40,73 @@ import org.slf4j.Logger;
import org.slf4j.helpers.MessageFormatter;
import java.io.Closeable;
-import java.util.ArrayList;
+import java.time.Duration;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
+import java.util.stream.Collectors;
import static org.apache.kafka.clients.consumer.internals.FetchUtils.requestMetadataUpdate;
/**
* {@code AbstractFetch} represents the basic state and logic for record fetching processing.
- * @param <K> Type for the message key
- * @param <V> Type for the message value
*/
-public abstract class AbstractFetch<K, V> implements Closeable {
+public abstract class AbstractFetch implements Closeable {
private final Logger log;
+ private final IdempotentCloser idempotentCloser = new IdempotentCloser();
protected final LogContext logContext;
- protected final ConsumerNetworkClient client;
protected final ConsumerMetadata metadata;
protected final SubscriptionState subscriptions;
- protected final FetchConfig<K, V> fetchConfig;
+ protected final FetchConfig fetchConfig;
protected final Time time;
protected final FetchMetricsManager metricsManager;
protected final FetchBuffer fetchBuffer;
protected final BufferSupplier decompressionBufferSupplier;
protected final Set<Integer> nodesWithPendingFetchRequests;
- protected final IdempotentCloser idempotentCloser = new IdempotentCloser();
private final Map<Integer, FetchSessionHandler> sessionHandlers;
public AbstractFetch(final LogContext logContext,
- final ConsumerNetworkClient client,
final ConsumerMetadata metadata,
final SubscriptionState subscriptions,
- final FetchConfig<K, V> fetchConfig,
+ final FetchConfig fetchConfig,
+ final FetchBuffer fetchBuffer,
final FetchMetricsManager metricsManager,
final Time time) {
this.log = logContext.logger(AbstractFetch.class);
this.logContext = logContext;
- this.client = client;
this.metadata = metadata;
this.subscriptions = subscriptions;
this.fetchConfig = fetchConfig;
+ this.fetchBuffer = fetchBuffer;
this.decompressionBufferSupplier = BufferSupplier.create();
- this.fetchBuffer = new FetchBuffer(logContext);
this.sessionHandlers = new HashMap<>();
this.nodesWithPendingFetchRequests = new HashSet<>();
this.metricsManager = metricsManager;
this.time = time;
}
+ /**
+ * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in
+ * reconnect backoff window following the disconnect).
+ *
+ * @param node {@link Node} to check for availability
+ * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time)
+ */
+ protected abstract boolean isUnavailable(Node node);
+
+ /**
+ * Checks for an authentication error on a given node and throws the exception if it exists.
+ *
+ * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown
+ * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node)
+ */
+ protected abstract void maybeThrowAuthFailure(Node node);
+
/**
* Return whether we have any completed fetches pending return to the user. This method is thread-safe. Has
* visibility for testing.
@@ -111,15 +126,15 @@ public abstract class AbstractFetch<K, V> implements Closeable {
}
/**
- * Implements the core logic for a successful fetch request/response.
+ * Implements the core logic for a successful fetch response.
*
* @param fetchTarget {@link Node} from which the fetch data was requested
* @param data {@link FetchSessionHandler.FetchRequestData} that represents the session data
* @param resp {@link ClientResponse} from which the {@link FetchResponse} will be retrieved
*/
- protected void handleFetchResponse(final Node fetchTarget,
- final FetchSessionHandler.FetchRequestData data,
- final ClientResponse resp) {
+ protected void handleFetchSuccess(final Node fetchTarget,
+ final FetchSessionHandler.FetchRequestData data,
+ final ClientResponse resp) {
try {
final FetchResponse response = (FetchResponse) resp.responseBody();
final FetchSessionHandler handler = sessionHandler(fetchTarget.id());
@@ -185,18 +200,20 @@ public abstract class AbstractFetch<K, V> implements Closeable {
metricsManager.recordLatency(resp.requestLatencyMs());
} finally {
- log.debug("Removing pending request for node {}", fetchTarget);
- nodesWithPendingFetchRequests.remove(fetchTarget.id());
+ removePendingFetchRequest(fetchTarget, data.metadata().sessionId());
}
}
/**
- * Implements the core logic for a failed fetch request/response.
+ * Implements the core logic for a failed fetch response.
*
* @param fetchTarget {@link Node} from which the fetch data was requested
- * @param t {@link Throwable} representing the error that resulted in the failure
+ * @param data {@link FetchSessionHandler.FetchRequestData} from request
+ * @param t {@link Throwable} representing the error that resulted in the failure
*/
- protected void handleFetchResponse(final Node fetchTarget, final Throwable t) {
+ protected void handleFetchFailure(final Node fetchTarget,
+ final FetchSessionHandler.FetchRequestData data,
+ final Throwable t) {
try {
final FetchSessionHandler handler = sessionHandler(fetchTarget.id());
@@ -205,25 +222,32 @@ public abstract class AbstractFetch<K, V> implements Closeable {
handler.sessionTopicPartitions().forEach(subscriptions::clearPreferredReadReplica);
}
} finally {
- log.debug("Removing pending request for node {}", fetchTarget);
- nodesWithPendingFetchRequests.remove(fetchTarget.id());
+ removePendingFetchRequest(fetchTarget, data.metadata().sessionId());
}
}
- protected void handleCloseFetchSessionResponse(final Node fetchTarget,
- final FetchSessionHandler.FetchRequestData data) {
+ protected void handleCloseFetchSessionSuccess(final Node fetchTarget,
+ final FetchSessionHandler.FetchRequestData data,
+ final ClientResponse ignored) {
int sessionId = data.metadata().sessionId();
+ removePendingFetchRequest(fetchTarget, sessionId);
log.debug("Successfully sent a close message for fetch session: {} to node: {}", sessionId, fetchTarget);
}
- public void handleCloseFetchSessionResponse(final Node fetchTarget,
- final FetchSessionHandler.FetchRequestData data,
- final Throwable t) {
+ public void handleCloseFetchSessionFailure(final Node fetchTarget,
+ final FetchSessionHandler.FetchRequestData data,
+ final Throwable t) {
int sessionId = data.metadata().sessionId();
- log.debug("Unable to a close message for fetch session: {} to node: {}. " +
+ removePendingFetchRequest(fetchTarget, sessionId);
+ log.debug("Unable to send a close message for fetch session: {} to node: {}. " +
"This may result in unnecessary fetch sessions at the broker.", sessionId, fetchTarget, t);
}
+ private void removePendingFetchRequest(Node fetchTarget, int sessionId) {
+ log.debug("Removing pending request for fetch session: {} for node: {}", sessionId, fetchTarget);
+ nodesWithPendingFetchRequests.remove(fetchTarget.id());
+ }
+
/**
* Creates a new {@link FetchRequest fetch request} in preparation for sending to the Kafka cluster.
*
@@ -317,9 +341,9 @@ public abstract class AbstractFetch<K, V> implements Closeable {
}
}
- private Map<Node, FetchSessionHandler.FetchRequestData> prepareCloseFetchSessionRequests() {
+ protected Map<Node, FetchSessionHandler.FetchRequestData> prepareCloseFetchSessionRequests() {
final Cluster cluster = metadata.fetch();
- Map<Node, FetchSessionHandler.Builder> fetchable = new LinkedHashMap<>();
+ Map<Node, FetchSessionHandler.Builder> fetchable = new HashMap<>();
try {
sessionHandlers.forEach((fetchTargetNodeId, sessionHandler) -> {
@@ -330,7 +354,7 @@ public abstract class AbstractFetch<K, V> implements Closeable {
// skip sending the close request.
final Node fetchTarget = cluster.nodeById(fetchTargetNodeId);
- if (fetchTarget == null || client.isUnavailable(fetchTarget)) {
+ if (fetchTarget == null || isUnavailable(fetchTarget)) {
log.debug("Skip sending close session request to broker {} since it is not reachable", fetchTarget);
return;
}
@@ -341,11 +365,7 @@ public abstract class AbstractFetch<K, V> implements Closeable {
sessionHandlers.clear();
}
- Map<Node, FetchSessionHandler.FetchRequestData> reqs = new LinkedHashMap<>();
- for (Map.Entry<Node, FetchSessionHandler.Builder> entry : fetchable.entrySet()) {
- reqs.put(entry.getKey(), entry.getValue().build());
- }
- return reqs;
+ return fetchable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build()));
}
/**
@@ -356,7 +376,7 @@ public abstract class AbstractFetch<K, V> implements Closeable {
// Update metrics in case there was an assignment change
metricsManager.maybeUpdateAssignment(subscriptions);
- Map<Node, FetchSessionHandler.Builder> fetchable = new LinkedHashMap<>();
+ Map<Node, FetchSessionHandler.Builder> fetchable = new HashMap<>();
long currentTimeMs = time.milliseconds();
Map<String, Uuid> topicIds = metadata.topicIds();
@@ -377,8 +397,8 @@ public abstract class AbstractFetch<K, V> implements Closeable {
// Use the preferred read replica if set, otherwise the partition's leader
Node node = selectReadReplica(partition, leaderOpt.get(), currentTimeMs);
- if (client.isUnavailable(node)) {
- client.maybeThrowAuthFailure(node);
+ if (isUnavailable(node)) {
+ maybeThrowAuthFailure(node);
// If we try to send during the reconnect backoff window, then the request is just
// going to be failed anyway before being sent, so skip sending the request for now
@@ -405,51 +425,7 @@ public abstract class AbstractFetch<K, V> implements Closeable {
}
}
- Map<Node, FetchSessionHandler.FetchRequestData> reqs = new LinkedHashMap<>();
- for (Map.Entry<Node, FetchSessionHandler.Builder> entry : fetchable.entrySet()) {
- reqs.put(entry.getKey(), entry.getValue().build());
- }
- return reqs;
- }
-
- protected void maybeCloseFetchSessions(final Timer timer) {
- final List<RequestFuture<ClientResponse>> requestFutures = new ArrayList<>();
- Map<Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareCloseFetchSessionRequests();
-
- for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequestMap.entrySet()) {
- final Node fetchTarget = entry.getKey();
- final FetchSessionHandler.FetchRequestData data = entry.getValue();
- final FetchRequest.Builder request = createFetchRequest(fetchTarget, data);
- final RequestFuture<ClientResponse> responseFuture = client.send(fetchTarget, request);
-
- responseFuture.addListener(new RequestFutureListener<ClientResponse>() {
- @Override
- public void onSuccess(ClientResponse value) {
- handleCloseFetchSessionResponse(fetchTarget, data);
- }
-
- @Override
- public void onFailure(RuntimeException e) {
- handleCloseFetchSessionResponse(fetchTarget, data, e);
- }
- });
-
- requestFutures.add(responseFuture);
- }
-
- // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until
- // all requests have received a response.
- while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) {
- client.poll(timer, null, true);
- }
-
- if (!requestFutures.stream().allMatch(RequestFuture::isDone)) {
- // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown
- // here.
- log.debug("All requests couldn't be sent in the specific timeout period {}ms. " +
- "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " +
- "KafkaConsumer.close(Duration timeout)", timer.timeoutMs());
- }
+ return fetchable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build()));
}
// Visible for testing
@@ -467,20 +443,29 @@ public abstract class AbstractFetch<K, V> implements Closeable {
// Visible for testing
protected void closeInternal(Timer timer) {
// we do not need to re-enable wake-ups since we are closing already
- client.disableWakeups();
- maybeCloseFetchSessions(timer);
Utils.closeQuietly(fetchBuffer, "fetchBuffer");
Utils.closeQuietly(decompressionBufferSupplier, "decompressionBufferSupplier");
}
public void close(final Timer timer) {
- idempotentCloser.close(() -> {
- closeInternal(timer);
- });
+ idempotentCloser.close(() -> closeInternal(timer));
}
@Override
public void close() {
- close(time.timer(0));
+ close(time.timer(Duration.ZERO));
+ }
+
+ /**
+ * Defines the contract for handling fetch responses from brokers.
+ * @param <T> Type of response, usually either {@link ClientResponse} or {@link Throwable}
+ */
+ @FunctionalInterface
+ protected interface ResponseHandler<T> {
+
+ /**
+ * Handle the response from the given {@link Node target}
+ */
+ void handle(Node target, FetchSessionHandler.FetchRequestData data, T response);
}
}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java
similarity index 61%
rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java
rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java
index 31c18731219..dc891a2539b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java
@@ -16,19 +16,28 @@
*/
package org.apache.kafka.clients.consumer.internals;
-import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
-import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
+import java.util.function.Supplier;
-import java.util.Queue;
+/**
+ * Simple {@link Supplier} that caches the initial creation of the object and stores it for later calls
+ * to {@link #get()}.
+ *
+ * <p/>
+ *
+ * <em>Note</em>: this class is not thread safe! Use only in contexts which are designed/guaranteed to be
+ * single-threaded.
+ */
+public abstract class CachedSupplier<T> implements Supplier<T> {
-public class ErrorEventHandler {
- private final Queue<BackgroundEvent> backgroundEventQueue;
+ private T result;
- public ErrorEventHandler(Queue<BackgroundEvent> backgroundEventQueue) {
- this.backgroundEventQueue = backgroundEventQueue;
- }
+ protected abstract T create();
+
+ @Override
+ public T get() {
+ if (result == null)
+ result = create();
- public void handle(Throwable e) {
- backgroundEventQueue.add(new ErrorBackgroundEvent(e));
+ return result;
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
index 65ac7c983c7..d3004dd9227 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
@@ -48,10 +48,12 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
+import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY;
+
public class CommitRequestManager implements RequestManager {
+
// TODO: current in ConsumerConfig but inaccessible in the internal package.
private static final String THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED = "internal.throw.on.fetch.stable.offset.unsupported";
- // TODO: We will need to refactor the subscriptionState
private final SubscriptionState subscriptions;
private final LogContext logContext;
private final Logger log;
@@ -96,17 +98,14 @@ public class CommitRequestManager implements RequestManager {
@Override
public NetworkClientDelegate.PollResult poll(final long currentTimeMs) {
// poll only when the coordinator node is known.
- if (!coordinatorRequestManager.coordinator().isPresent()) {
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList());
- }
+ if (!coordinatorRequestManager.coordinator().isPresent())
+ return EMPTY;
maybeAutoCommit(this.subscriptions.allConsumed());
- if (!pendingRequests.hasUnsentRequests()) {
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList());
- }
+ if (!pendingRequests.hasUnsentRequests())
+ return EMPTY;
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE,
- Collections.unmodifiableList(pendingRequests.drain(currentTimeMs)));
+ return new NetworkClientDelegate.PollResult(pendingRequests.drain(currentTimeMs));
}
public void maybeAutoCommit(final Map<TopicPartition, OffsetAndMetadata> offsets) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java
index 7a8ee105157..8959345bffd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java
@@ -53,7 +53,7 @@ import java.util.Set;
/**
* {@link CompletedFetch} represents a {@link RecordBatch batch} of {@link Record records} that was returned from the
* broker via a {@link FetchRequest}. It contains logic to maintain state between calls to
- * {@link #fetchRecords(FetchConfig, int)}.
+ * {@link #fetchRecords(FetchConfig, Deserializers, int)}.
*/
public class CompletedFetch {
@@ -135,7 +135,8 @@ public class CompletedFetch {
/**
* Draining a {@link CompletedFetch} will signal that the data has been consumed and the underlying resources
* are closed. This is somewhat analogous to {@link Closeable#close() closing}, though no error will result if a
- * caller invokes {@link #fetchRecords(FetchConfig, int)}; an empty {@link List list} will be returned instead.
+ * caller invokes {@link #fetchRecords(FetchConfig, Deserializers, int)}; an empty {@link List list} will be
+ * returned instead.
*/
void drain() {
if (!isConsumed) {
@@ -151,7 +152,7 @@ public class CompletedFetch {
}
}
- private <K, V> void maybeEnsureValid(FetchConfig<K, V> fetchConfig, RecordBatch batch) {
+ private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) {
if (fetchConfig.checkCrcs && batch.magic() >= RecordBatch.MAGIC_VALUE_V2) {
try {
batch.ensureValid();
@@ -162,7 +163,7 @@ public class CompletedFetch {
}
}
- private <K, V> void maybeEnsureValid(FetchConfig<K, V> fetchConfig, Record record) {
+ private void maybeEnsureValid(FetchConfig fetchConfig, Record record) {
if (fetchConfig.checkCrcs) {
try {
record.ensureValid();
@@ -180,7 +181,7 @@ public class CompletedFetch {
}
}
- private <K, V> Record nextFetchedRecord(FetchConfig<K, V> fetchConfig) {
+ private Record nextFetchedRecord(FetchConfig fetchConfig) {
while (true) {
if (records == null || !records.hasNext()) {
maybeCloseRecordStream();
@@ -245,11 +246,14 @@ public class CompletedFetch {
* {@link Deserializer deserialization} of the {@link Record record's} key and value are performed in
* this step.
*
- * @param fetchConfig {@link FetchConfig Configuration} to use, including, but not limited to, {@link Deserializer}s
+ * @param fetchConfig {@link FetchConfig Configuration} to use
+ * @param deserializers {@link Deserializer}s to use to convert the raw bytes to the expected key and value types
* @param maxRecords The number of records to return; the number returned may be {@code 0 <= maxRecords}
* @return {@link ConsumerRecord Consumer records}
*/
- <K, V> List<ConsumerRecord<K, V>> fetchRecords(FetchConfig<K, V> fetchConfig, int maxRecords) {
+ <K, V> List<ConsumerRecord<K, V>> fetchRecords(FetchConfig fetchConfig,
+ Deserializers<K, V> deserializers,
+ int maxRecords) {
// Error when fetching the next record before deserialization.
if (corruptLastRecord)
throw new KafkaException("Received exception when fetching the next record from " + partition
@@ -276,7 +280,7 @@ public class CompletedFetch {
Optional<Integer> leaderEpoch = maybeLeaderEpoch(currentBatch.partitionLeaderEpoch());
TimestampType timestampType = currentBatch.timestampType();
- ConsumerRecord<K, V> record = parseRecord(fetchConfig, partition, leaderEpoch, timestampType, lastRecord);
+ ConsumerRecord<K, V> record = parseRecord(deserializers, partition, leaderEpoch, timestampType, lastRecord);
records.add(record);
recordsRead++;
bytesRead += lastRecord.sizeInBytes();
@@ -302,7 +306,7 @@ public class CompletedFetch {
/**
* Parse the record entry, deserializing the key / value fields if necessary
*/
- <K, V> ConsumerRecord<K, V> parseRecord(FetchConfig<K, V> fetchConfig,
+ <K, V> ConsumerRecord<K, V> parseRecord(Deserializers<K, V> deserializers,
TopicPartition partition,
Optional<Integer> leaderEpoch,
TimestampType timestampType,
@@ -312,16 +316,16 @@ public class CompletedFetch {
long timestamp = record.timestamp();
Headers headers = new RecordHeaders(record.headers());
ByteBuffer keyBytes = record.key();
- K key = keyBytes == null ? null : fetchConfig.deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes);
+ K key = keyBytes == null ? null : deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes);
ByteBuffer valueBytes = record.value();
- V value = valueBytes == null ? null : fetchConfig.deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes);
+ V value = valueBytes == null ? null : deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes);
return new ConsumerRecord<>(partition.topic(), partition.partition(), offset,
timestamp, timestampType,
keyBytes == null ? ConsumerRecord.NULL_SIZE : keyBytes.remaining(),
valueBytes == null ? ConsumerRecord.NULL_SIZE : valueBytes.remaining(),
key, value, headers, leaderEpoch);
} catch (RuntimeException e) {
- log.error("Deserializers with error: {}", fetchConfig.deserializers);
+ log.error("Deserializers with error: {}", deserializers);
throw new RecordDeserializationException(partition, record.offset(),
"Error deserializing key/value for partition " + partition +
" at offset " + record.offset() + ". If needed, please seek past the record to continue consumption.", e);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index b906a02fef9..bdcbfc39dfc 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -955,10 +955,16 @@ public final class ConsumerCoordinator extends AbstractCoordinator {
* @param timer Timer bounding how long this method can block
* @return true iff the operation completed within the timeout
*/
- public boolean refreshCommittedOffsetsIfNeeded(Timer timer) {
+ public boolean initWithCommittedOffsetsIfNeeded(Timer timer) {
final Set<TopicPartition> initializingPartitions = subscriptions.initializingPartitions();
final Map<TopicPartition, OffsetAndMetadata> offsets = fetchCommittedOffsets(initializingPartitions, timer);
- return refreshCommittedOffsets(offsets, this.metadata, this.subscriptions);
+
+ // "offsets" will be null if the offset fetch requests did not receive responses within the given timeout
+ if (offsets == null)
+ return false;
+
+ refreshCommittedOffsets(offsets, this.metadata, this.subscriptions);
+ return true;
}
/**
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java
new file mode 100644
index 00000000000..77a2952d1b2
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import org.apache.kafka.clients.KafkaClient;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.internals.IdempotentCloser;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Timer;
+import org.slf4j.Logger;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS;
+import static org.apache.kafka.common.utils.Utils.closeQuietly;
+
+/**
+ * Background thread runnable that consumes {@link ApplicationEvent} and produces {@link BackgroundEvent}. It
+ * uses an event loop to consume and produce events, and poll the network client to handle network IO.
+ */
+public class ConsumerNetworkThread extends KafkaThread implements Closeable {
+
+ private static final long MAX_POLL_TIMEOUT_MS = 5000;
+ private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread";
+ private final Time time;
+ private final Logger log;
+ private final Supplier<ApplicationEventProcessor> applicationEventProcessorSupplier;
+ private final Supplier<NetworkClientDelegate> networkClientDelegateSupplier;
+ private final Supplier<RequestManagers> requestManagersSupplier;
+ private ApplicationEventProcessor applicationEventProcessor;
+ private NetworkClientDelegate networkClientDelegate;
+ private RequestManagers requestManagers;
+ private volatile boolean running;
+ private final IdempotentCloser closer = new IdempotentCloser();
+ private volatile Duration closeTimeout = Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS);
+
+ public ConsumerNetworkThread(LogContext logContext,
+ Time time,
+ Supplier<ApplicationEventProcessor> applicationEventProcessorSupplier,
+ Supplier<NetworkClientDelegate> networkClientDelegateSupplier,
+ Supplier<RequestManagers> requestManagersSupplier) {
+ super(BACKGROUND_THREAD_NAME, true);
+ this.time = time;
+ this.log = logContext.logger(getClass());
+ this.applicationEventProcessorSupplier = applicationEventProcessorSupplier;
+ this.networkClientDelegateSupplier = networkClientDelegateSupplier;
+ this.requestManagersSupplier = requestManagersSupplier;
+ }
+
+ @Override
+ public void run() {
+ closer.assertOpen("Consumer network thread is already closed");
+ running = true;
+
+ try {
+ log.debug("Consumer network thread started");
+
+ // Wait until we're securely in the background network thread to initialize these objects...
+ initializeResources();
+
+ while (running) {
+ try {
+ runOnce();
+ } catch (final WakeupException e) {
+ log.debug("WakeupException caught, consumer network thread won't be interrupted");
+ // swallow the wakeup exception to prevent killing the thread.
+ }
+ }
+ } catch (final Throwable t) {
+ log.error("The consumer network thread failed due to unexpected error", t);
+ throw new KafkaException(t);
+ } finally {
+ cleanup();
+ }
+ }
+
+ void initializeResources() {
+ applicationEventProcessor = applicationEventProcessorSupplier.get();
+ networkClientDelegate = networkClientDelegateSupplier.get();
+ requestManagers = requestManagersSupplier.get();
+ }
+
+ /**
+ * Poll and process the {@link ApplicationEvent application events}. It performs the following tasks:
+ *
+ * <ol>
+ * <li>
+ * Drains and processes all the events from the application thread's application event queue via
+ * {@link ApplicationEventProcessor}
+ * </li>
+ * <li>
+ * Iterate through the {@link RequestManager} list and invoke {@link RequestManager#poll(long)} to get
+ * the {@link NetworkClientDelegate.UnsentRequest} list and the poll time for the network poll
+ * </li>
+ * <li>
+ * Stage each {@link AbstractRequest.Builder request} to be sent via
+ * {@link NetworkClientDelegate#addAll(List)}
+ * </li>
+ * <li>
+ * Poll the client via {@link KafkaClient#poll(long, long)} to send the requests, as well as
+ * retrieve any available responses
+ * </li>
+ * </ol>
+ */
+ void runOnce() {
+ // If there are errors processing any events, the error will be thrown immediately. This will have
+ // the effect of closing the background thread.
+ applicationEventProcessor.process();
+
+ final long currentTimeMs = time.milliseconds();
+ final long pollWaitTimeMs = requestManagers.entries().stream()
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .map(rm -> rm.poll(currentTimeMs))
+ .map(networkClientDelegate::addAll)
+ .reduce(MAX_POLL_TIMEOUT_MS, Math::min);
+ networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs);
+ }
+
+ /**
+ * Performs any network I/O that is needed at the time of close for the consumer:
+ *
+ * <ol>
+ * <li>
+ * Iterate through the {@link RequestManager} list and invoke {@link RequestManager#pollOnClose()}
+ * to get the {@link NetworkClientDelegate.UnsentRequest} list and the poll time for the network poll
+ * </li>
+ * <li>
+ * Stage each {@link AbstractRequest.Builder request} to be sent via
+ * {@link NetworkClientDelegate#addAll(List)}
+ * </li>
+ * <li>
+ * {@link KafkaClient#poll(long, long) Poll the client} to send the requests, as well as
+ * retrieve any available responses
+ * </li>
+ * <li>
+ * Continuously {@link KafkaClient#poll(long, long) poll the client} as long as the
+ * {@link Timer#notExpired() timer hasn't expired} to retrieve the responses
+ * </li>
+ * </ol>
+ */
+ // Visible for testing
+ static void runAtClose(final Collection<Optional<? extends RequestManager>> requestManagers,
+ final NetworkClientDelegate networkClientDelegate,
+ final Timer timer) {
+ // These are the optional outgoing requests at the
+ List<NetworkClientDelegate.PollResult> pollResults = requestManagers.stream()
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .map(RequestManager::pollOnClose)
+ .collect(Collectors.toList());
+ long pollWaitTimeMs = pollResults.stream()
+ .map(networkClientDelegate::addAll)
+ .reduce(MAX_POLL_TIMEOUT_MS, Math::min);
+ pollWaitTimeMs = Math.min(pollWaitTimeMs, timer.remainingMs());
+ networkClientDelegate.poll(pollWaitTimeMs, timer.currentTimeMs());
+ timer.update();
+
+ List<Future<?>> requestFutures = pollResults.stream()
+ .flatMap(fads -> fads.unsentRequests.stream())
+ .map(NetworkClientDelegate.UnsentRequest::future)
+ .collect(Collectors.toList());
+
+ // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until
+ // all requests have received a response.
+ while (timer.notExpired() && !requestFutures.stream().allMatch(Future::isDone)) {
+ networkClientDelegate.poll(timer.remainingMs(), timer.currentTimeMs());
+ timer.update();
+ }
+ }
+
+ public boolean isRunning() {
+ return running;
+ }
+
+ public void wakeup() {
+ // The network client can be null if the initializeResources method has not yet been called.
+ if (networkClientDelegate != null)
+ networkClientDelegate.wakeup();
+ }
+
+ @Override
+ public void close() {
+ close(closeTimeout);
+ }
+
+ public void close(final Duration timeout) {
+ Objects.requireNonNull(timeout, "Close timeout for consumer network thread must be non-null");
+
+ closer.close(
+ () -> closeInternal(timeout),
+ () -> log.warn("The consumer network thread was already closed")
+ );
+ }
+
+ /**
+ * Starts the closing process.
+ *
+ * <p/>
+ *
+ * This method is called from the application thread, but our resources are owned by the network thread. As such,
+ * we don't actually close any of those resources here, immediately, on the application thread. Instead, we just
+ * update our internal state on the application thread. When the network thread next
+ * {@link #run() executes its loop}, it will notice that state, cease processing any further events, and begin
+ * {@link #cleanup() closing its resources}.
+ *
+ * <p/>
+ *
+ * This method will wait (i.e. block the application thread) for up to the duration of the given timeout to give
+ * the network thread the time to close down cleanly.
+ *
+ * @param timeout Upper bound of time to wait for the network thread to close its resources
+ */
+ private void closeInternal(final Duration timeout) {
+ long timeoutMs = timeout.toMillis();
+ log.trace("Signaling the consumer network thread to close in {}ms", timeoutMs);
+ running = false;
+ closeTimeout = timeout;
+ wakeup();
+
+ if (timeoutMs > 0) {
+ try {
+ join(timeoutMs);
+ } catch (InterruptedException e) {
+ log.error("Interrupted while waiting for consumer network thread to complete", e);
+ }
+ }
+ }
+
+ void cleanup() {
+ log.trace("Closing the consumer network thread");
+ Timer timer = time.timer(closeTimeout);
+ runAtClose(requestManagers.entries(), networkClientDelegate, timer);
+ closeQuietly(requestManagers, "request managers");
+ closeQuietly(networkClientDelegate, "network client delegate");
+ closeQuietly(applicationEventProcessor, "application event processor");
+ log.debug("Closed the consumer network thread");
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java
index 68d2458e28d..92b098213b0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java
@@ -56,6 +56,7 @@ import java.util.concurrent.TimeUnit;
public final class ConsumerUtils {
+ public static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000;
public static final String CONSUMER_JMX_PREFIX = "kafka.consumer";
public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer";
@@ -142,12 +143,6 @@ public final class ConsumerUtils {
return new FetchMetricsManager(metrics, metricsRegistry);
}
- public static <K, V> FetchConfig<K, V> createFetchConfig(ConsumerConfig config,
- Deserializers<K, V> deserializers) {
- IsolationLevel isolationLevel = configuredIsolationLevel(config);
- return new FetchConfig<>(config, deserializers, isolationLevel);
- }
-
@SuppressWarnings("unchecked")
public static <K, V> List<ConsumerInterceptor<K, V>> configuredConsumerInterceptors(ConsumerConfig config) {
return (List<ConsumerInterceptor<K, V>>) ClientUtils.configuredInterceptors(config, ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptor.class);
@@ -168,14 +163,10 @@ public final class ConsumerUtils {
* committed offsets' metadata.
* @param subscriptions Subscription state to update, setting partitions' offsets to the
* committed offsets.
- * @return False if null <code>offsetsAndMetadata</code> is provided, indicating that the
- * refresh operation could not be performed. True in any other case.
*/
- public static boolean refreshCommittedOffsets(final Map<TopicPartition, OffsetAndMetadata> offsetsAndMetadata,
- final ConsumerMetadata metadata,
- final SubscriptionState subscriptions) {
- if (offsetsAndMetadata == null) return false;
-
+ public static void refreshCommittedOffsets(final Map<TopicPartition, OffsetAndMetadata> offsetsAndMetadata,
+ final ConsumerMetadata metadata,
+ final SubscriptionState subscriptions) {
for (final Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsetsAndMetadata.entrySet()) {
final TopicPartition tp = entry.getKey();
final OffsetAndMetadata offsetAndMetadata = entry.getValue();
@@ -200,7 +191,6 @@ public final class ConsumerUtils {
}
}
}
- return true;
}
public static <T> T getResult(CompletableFuture<T> future, Timer timer) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java
index c67bbeaa34e..3efe45499f2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java
@@ -16,6 +16,9 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
+import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.errors.GroupAuthorizationException;
import org.apache.kafka.common.errors.RetriableException;
@@ -28,10 +31,11 @@ import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
import org.slf4j.Logger;
-import java.util.Collections;
import java.util.Objects;
import java.util.Optional;
+import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY;
+
/**
* This is responsible for timing to send the next {@link FindCoordinatorRequest} based on the following criteria:
* <p/>
@@ -49,7 +53,7 @@ public class CoordinatorRequestManager implements RequestManager {
private static final long COORDINATOR_DISCONNECT_LOGGING_INTERVAL_MS = 60 * 1000;
private final Time time;
private final Logger log;
- private final ErrorEventHandler nonRetriableErrorHandler;
+ private final BackgroundEventHandler backgroundEventHandler;
private final String groupId;
private final RequestState coordinatorRequestState;
@@ -62,13 +66,13 @@ public class CoordinatorRequestManager implements RequestManager {
final LogContext logContext,
final long retryBackoffMs,
final long retryBackoffMaxMs,
- final ErrorEventHandler errorHandler,
+ final BackgroundEventHandler errorHandler,
final String groupId
) {
Objects.requireNonNull(groupId);
this.time = time;
this.log = logContext.logger(this.getClass());
- this.nonRetriableErrorHandler = errorHandler;
+ this.backgroundEventHandler = errorHandler;
this.groupId = groupId;
this.coordinatorRequestState = new RequestState(
logContext,
@@ -90,18 +94,15 @@ public class CoordinatorRequestManager implements RequestManager {
*/
@Override
public NetworkClientDelegate.PollResult poll(final long currentTimeMs) {
- if (this.coordinator != null) {
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList());
- }
+ if (this.coordinator != null)
+ return EMPTY;
if (coordinatorRequestState.canSendRequest(currentTimeMs)) {
NetworkClientDelegate.UnsentRequest request = makeFindCoordinatorRequest(currentTimeMs);
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.singletonList(request));
+ return new NetworkClientDelegate.PollResult(request);
}
- return new NetworkClientDelegate.PollResult(
- coordinatorRequestState.remainingBackoffMs(currentTimeMs),
- Collections.emptyList());
+ return new NetworkClientDelegate.PollResult(coordinatorRequestState.remainingBackoffMs(currentTimeMs));
}
private NetworkClientDelegate.UnsentRequest makeFindCoordinatorRequest(final long currentTimeMs) {
@@ -175,12 +176,13 @@ public class CoordinatorRequestManager implements RequestManager {
if (exception == Errors.GROUP_AUTHORIZATION_FAILED.exception()) {
log.debug("FindCoordinator request failed due to authorization error {}", exception.getMessage());
- nonRetriableErrorHandler.handle(GroupAuthorizationException.forGroupId(this.groupId));
+ KafkaException groupAuthorizationException = GroupAuthorizationException.forGroupId(this.groupId);
+ backgroundEventHandler.add(new ErrorBackgroundEvent(groupAuthorizationException));
return;
}
log.warn("FindCoordinator request failed due to fatal exception", exception);
- nonRetriableErrorHandler.handle(exception);
+ backgroundEventHandler.add(new ErrorBackgroundEvent(exception));
}
/**
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java
deleted file mode 100644
index 74322e6fec0..00000000000
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals;
-
-import org.apache.kafka.clients.ApiVersions;
-import org.apache.kafka.clients.ClientUtils;
-import org.apache.kafka.clients.GroupRebalanceConfig;
-import org.apache.kafka.clients.NetworkClient;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
-import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
-import org.apache.kafka.common.KafkaException;
-import org.apache.kafka.common.errors.WakeupException;
-import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.utils.KafkaThread;
-import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Utils;
-import org.slf4j.Logger;
-
-import java.util.LinkedList;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.BlockingQueue;
-
-import static java.util.Objects.requireNonNull;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel;
-
-/**
- * Background thread runnable that consumes {@code ApplicationEvent} and
- * produces {@code BackgroundEvent}. It uses an event loop to consume and
- * produce events, and poll the network client to handle network IO.
- * <p/>
- * It holds a reference to the {@link SubscriptionState}, which is
- * initialized by the polling thread.
- * <p/>
- * For processing application events that have been submitted to the
- * {@link #applicationEventQueue}, this relies on an {@link ApplicationEventProcessor}. Processing includes generating requests and
- * handling responses with the appropriate {@link RequestManager}. The network operations for
- * actually sending the requests is delegated to the {@link NetworkClientDelegate}
- * </li>
- */
-public class DefaultBackgroundThread extends KafkaThread {
- private static final long MAX_POLL_TIMEOUT_MS = 5000;
- private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread";
- private final Time time;
- private final Logger log;
- private final BlockingQueue<ApplicationEvent> applicationEventQueue;
- private final BlockingQueue<BackgroundEvent> backgroundEventQueue;
- private final ConsumerMetadata metadata;
- private final ConsumerConfig config;
- // empty if groupId is null
- private final ApplicationEventProcessor applicationEventProcessor;
- private final NetworkClientDelegate networkClientDelegate;
- private final ErrorEventHandler errorEventHandler;
- private final GroupState groupState;
- private boolean running;
-
- private final RequestManagers requestManagers;
-
- // Visible for testing
- @SuppressWarnings("ParameterNumber")
- DefaultBackgroundThread(final Time time,
- final ConsumerConfig config,
- final LogContext logContext,
- final BlockingQueue<ApplicationEvent> applicationEventQueue,
- final BlockingQueue<BackgroundEvent> backgroundEventQueue,
- final ErrorEventHandler errorEventHandler,
- final ApplicationEventProcessor processor,
- final ConsumerMetadata metadata,
- final NetworkClientDelegate networkClient,
- final GroupState groupState,
- final CoordinatorRequestManager coordinatorManager,
- final CommitRequestManager commitRequestManager,
- final OffsetsRequestManager offsetsRequestManager,
- final TopicMetadataRequestManager topicMetadataRequestManager,
- final HeartbeatRequestManager heartbeatRequestManager) {
- super(BACKGROUND_THREAD_NAME, true);
- this.time = time;
- this.running = true;
- this.log = logContext.logger(getClass());
- this.applicationEventQueue = applicationEventQueue;
- this.backgroundEventQueue = backgroundEventQueue;
- this.applicationEventProcessor = processor;
- this.config = config;
- this.metadata = metadata;
- this.networkClientDelegate = networkClient;
- this.errorEventHandler = errorEventHandler;
- this.groupState = groupState;
- this.requestManagers = new RequestManagers(
- offsetsRequestManager,
- topicMetadataRequestManager,
- Optional.ofNullable(coordinatorManager),
- Optional.ofNullable(commitRequestManager),
- Optional.ofNullable(heartbeatRequestManager));
- }
-
- public DefaultBackgroundThread(final Time time,
- final ConsumerConfig config,
- final GroupRebalanceConfig rebalanceConfig,
- final LogContext logContext,
- final BlockingQueue<ApplicationEvent> applicationEventQueue,
- final BlockingQueue<BackgroundEvent> backgroundEventQueue,
- final ConsumerMetadata metadata,
- final SubscriptionState subscriptionState,
- final ApiVersions apiVersions,
- final Metrics metrics,
- final Sensor fetcherThrottleTimeSensor) {
- super(BACKGROUND_THREAD_NAME, true);
- requireNonNull(config);
- requireNonNull(rebalanceConfig);
- requireNonNull(logContext);
- requireNonNull(applicationEventQueue);
- requireNonNull(backgroundEventQueue);
- requireNonNull(metadata);
- requireNonNull(subscriptionState);
- try {
- this.time = time;
- this.log = logContext.logger(getClass());
- this.applicationEventQueue = applicationEventQueue;
- this.backgroundEventQueue = backgroundEventQueue;
- this.config = config;
- this.metadata = metadata;
- final NetworkClient networkClient = ClientUtils.createNetworkClient(config,
- metrics,
- CONSUMER_METRIC_GROUP_PREFIX,
- logContext,
- apiVersions,
- time,
- CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION,
- metadata,
- fetcherThrottleTimeSensor);
- this.networkClientDelegate = new NetworkClientDelegate(
- this.time,
- this.config,
- logContext,
- networkClient);
- this.running = true;
- this.errorEventHandler = new ErrorEventHandler(this.backgroundEventQueue);
- this.groupState = new GroupState(rebalanceConfig);
- long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
- long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG);
- final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG);
-
- OffsetsRequestManager offsetsRequestManager =
- new OffsetsRequestManager(
- subscriptionState,
- metadata,
- configuredIsolationLevel(config),
- time,
- retryBackoffMs,
- requestTimeoutMs,
- apiVersions,
- networkClientDelegate,
- logContext);
- CoordinatorRequestManager coordinatorRequestManager = null;
- CommitRequestManager commitRequestManager = null;
- TopicMetadataRequestManager topicMetadataRequestManger = new TopicMetadataRequestManager(
- logContext,
- config);
- HeartbeatRequestManager heartbeatRequestManager = null;
-
- // TODO: consolidate groupState and memberState
- if (groupState.groupId != null) {
- coordinatorRequestManager = new CoordinatorRequestManager(
- this.time,
- logContext,
- retryBackoffMs,
- retryBackoffMaxMs,
- this.errorEventHandler,
- groupState.groupId);
- commitRequestManager = new CommitRequestManager(
- this.time,
- logContext,
- subscriptionState,
- config,
- coordinatorRequestManager,
- groupState);
- MembershipManager membershipManager = new MembershipManagerImpl(groupState.groupId, logContext);
- heartbeatRequestManager = new HeartbeatRequestManager(
- this.time,
- logContext,
- config,
- coordinatorRequestManager,
- subscriptionState,
- membershipManager,
- errorEventHandler);
- }
-
- this.requestManagers = new RequestManagers(
- offsetsRequestManager,
- topicMetadataRequestManger,
- Optional.ofNullable(coordinatorRequestManager),
- Optional.ofNullable(commitRequestManager),
- Optional.ofNullable(heartbeatRequestManager));
- this.applicationEventProcessor = new ApplicationEventProcessor(
- backgroundEventQueue,
- requestManagers,
- metadata);
- } catch (final Exception e) {
- close();
- throw new KafkaException("Failed to construct background processor", e.getCause());
- }
- }
-
- @Override
- public void run() {
- try {
- log.debug("Background thread started");
- while (running) {
- try {
- runOnce();
- } catch (final WakeupException e) {
- log.debug("WakeupException caught, background thread won't be interrupted");
- // swallow the wakeup exception to prevent killing the background thread.
- }
- }
- } catch (final Throwable t) {
- log.error("The background thread failed due to unexpected error", t);
- throw new KafkaException(t);
- } finally {
- close();
- log.debug("{} closed", getClass());
- }
- }
-
- /**
- * Poll and process an {@link ApplicationEvent}. It performs the following tasks:
- * 1. Drains and try to process all the requests in the queue.
- * 2. Iterate through the registry, poll, and get the next poll time for the network poll
- * 3. Poll the networkClient to send and retrieve the response.
- */
- void runOnce() {
- if (!applicationEventQueue.isEmpty()) {
- LinkedList<ApplicationEvent> res = new LinkedList<>();
- this.applicationEventQueue.drainTo(res);
-
- for (ApplicationEvent event : res) {
- log.debug("Consuming application event: {}", event);
- Objects.requireNonNull(event);
- applicationEventProcessor.process(event);
- }
- }
-
- final long currentTimeMs = time.milliseconds();
- final long pollWaitTimeMs = requestManagers.entries().stream()
- .filter(Optional::isPresent)
- .map(m -> m.get().poll(currentTimeMs))
- .map(this::handlePollResult)
- .reduce(MAX_POLL_TIMEOUT_MS, Math::min);
- networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs);
- }
-
- long handlePollResult(NetworkClientDelegate.PollResult res) {
- if (!res.unsentRequests.isEmpty()) {
- networkClientDelegate.addAll(res.unsentRequests);
- }
- return res.timeUntilNextPollMs;
- }
-
- public boolean isRunning() {
- return this.running;
- }
-
- public final void wakeup() {
- networkClientDelegate.wakeup();
- }
-
- public final void close() {
- this.running = false;
- this.wakeup();
- Utils.closeQuietly(networkClientDelegate, "network client utils");
- Utils.closeQuietly(metadata, "consumer metadata client");
- }
-}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java
deleted file mode 100644
index 4d620a4ffac..00000000000
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals;
-
-import org.apache.kafka.clients.ApiVersions;
-import org.apache.kafka.clients.ClientUtils;
-import org.apache.kafka.clients.GroupRebalanceConfig;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
-import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.EventHandler;
-import org.apache.kafka.common.internals.ClusterResourceListeners;
-import org.apache.kafka.common.metrics.Metrics;
-import org.apache.kafka.common.metrics.Sensor;
-import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.common.utils.Timer;
-
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * An {@code EventHandler} that uses a single background thread to consume {@code ApplicationEvent} and produce
- * {@code BackgroundEvent} from the {@link DefaultBackgroundThread}.
- */
-public class DefaultEventHandler implements EventHandler {
-
- private final BlockingQueue<ApplicationEvent> applicationEventQueue;
- private final BlockingQueue<BackgroundEvent> backgroundEventQueue;
- private final DefaultBackgroundThread backgroundThread;
-
- public DefaultEventHandler(final ConsumerConfig config,
- final GroupRebalanceConfig groupRebalanceConfig,
- final LogContext logContext,
- final SubscriptionState subscriptionState,
- final ApiVersions apiVersions,
- final Metrics metrics,
- final ClusterResourceListeners clusterResourceListeners,
- final Sensor fetcherThrottleTimeSensor) {
- this(Time.SYSTEM,
- config,
- groupRebalanceConfig,
- logContext,
- new LinkedBlockingQueue<>(),
- new LinkedBlockingQueue<>(),
- subscriptionState,
- apiVersions,
- metrics,
- clusterResourceListeners,
- fetcherThrottleTimeSensor);
- }
-
- public DefaultEventHandler(final Time time,
- final ConsumerConfig config,
- final GroupRebalanceConfig groupRebalanceConfig,
- final LogContext logContext,
- final BlockingQueue<ApplicationEvent> applicationEventQueue,
- final BlockingQueue<BackgroundEvent> backgroundEventQueue,
- final SubscriptionState subscriptionState,
- final ApiVersions apiVersions,
- final Metrics metrics,
- final ClusterResourceListeners clusterResourceListeners,
- final Sensor fetcherThrottleTimeSensor) {
- this.applicationEventQueue = applicationEventQueue;
- this.backgroundEventQueue = backgroundEventQueue;
-
- // Bootstrap a metadata object with the bootstrap server IP address, which will be used once for the
- // subsequent metadata refresh once the background thread has started up.
- final ConsumerMetadata metadata = new ConsumerMetadata(config,
- subscriptionState,
- logContext,
- clusterResourceListeners);
- final List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config);
- metadata.bootstrap(addresses);
-
- this.backgroundThread = new DefaultBackgroundThread(
- time,
- config,
- groupRebalanceConfig,
- logContext,
- this.applicationEventQueue,
- this.backgroundEventQueue,
- metadata,
- subscriptionState,
- apiVersions,
- metrics,
- fetcherThrottleTimeSensor);
- backgroundThread.start();
- }
-
- // VisibleForTesting
- DefaultEventHandler(final DefaultBackgroundThread backgroundThread,
- final BlockingQueue<ApplicationEvent> applicationEventQueue,
- final BlockingQueue<BackgroundEvent> backgroundEventQueue) {
- this.backgroundThread = backgroundThread;
- this.applicationEventQueue = applicationEventQueue;
- this.backgroundEventQueue = backgroundEventQueue;
- backgroundThread.start();
- }
-
- @Override
- public Optional<BackgroundEvent> poll() {
- return Optional.ofNullable(backgroundEventQueue.poll());
- }
-
- @Override
- public boolean isEmpty() {
- return backgroundEventQueue.isEmpty();
- }
-
- @Override
- public boolean add(final ApplicationEvent event) {
- backgroundThread.wakeup();
- return applicationEventQueue.add(event);
- }
-
- @Override
- public <T> T addAndGet(final CompletableApplicationEvent<T> event, final Timer timer) {
- Objects.requireNonNull(event, "CompletableApplicationEvent provided to addAndGet must be non-null");
- Objects.requireNonNull(timer, "Timer provided to addAndGet must be non-null");
- add(event);
- return event.get(timer);
- }
-
- public void close() {
- try {
- backgroundThread.close();
- } catch (final Exception e) {
- throw new RuntimeException(e);
- }
- }
-}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java
index 80ba5cb7d9e..5de2a888775 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java
@@ -75,4 +75,12 @@ public class Deserializers<K, V> implements AutoCloseable {
throw new KafkaException("Failed to close deserializers", exception);
}
}
+
+ @Override
+ public String toString() {
+ return "Deserializers{" +
+ "keyDeserializer=" + keyDeserializer +
+ ", valueDeserializer=" + valueDeserializer +
+ '}';
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java
index d82b37ad99a..de7f88ab725 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java
@@ -17,15 +17,21 @@
package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.InterruptException;
import org.apache.kafka.common.internals.IdempotentCloser;
import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Timer;
import org.slf4j.Logger;
-import java.io.Closeable;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Predicate;
/**
@@ -35,12 +41,15 @@ import java.util.function.Predicate;
*
* <p/>
*
- * <em>Note</em>: this class is not thread-safe and is intended to only be used from a single thread.
+ * <em>Note</em>: this class is thread-safe with the intention that {@link CompletedFetch the data} will be
+ * "produced" by a background thread and consumed by the application thread.
*/
-public class FetchBuffer implements Closeable {
+public class FetchBuffer implements AutoCloseable {
private final Logger log;
private final ConcurrentLinkedQueue<CompletedFetch> completedFetches;
+ private final Lock lock;
+ private final Condition notEmptyCondition;
private final IdempotentCloser idempotentCloser = new IdempotentCloser();
private CompletedFetch nextInLineFetch;
@@ -48,6 +57,8 @@ public class FetchBuffer implements Closeable {
public FetchBuffer(final LogContext logContext) {
this.log = logContext.logger(FetchBuffer.class);
this.completedFetches = new ConcurrentLinkedQueue<>();
+ this.lock = new ReentrantLock();
+ this.notEmptyCondition = lock.newCondition();
}
/**
@@ -56,7 +67,12 @@ public class FetchBuffer implements Closeable {
* @return {@code true} if the buffer is empty, {@code false} otherwise
*/
boolean isEmpty() {
- return completedFetches.isEmpty();
+ try {
+ lock.lock();
+ return completedFetches.isEmpty();
+ } finally {
+ lock.unlock();
+ }
}
/**
@@ -66,31 +82,107 @@ public class FetchBuffer implements Closeable {
* @return {@code true} if there are completed fetches that match the {@link Predicate}, {@code false} otherwise
*/
boolean hasCompletedFetches(Predicate<CompletedFetch> predicate) {
- return completedFetches.stream().anyMatch(predicate);
+ try {
+ lock.lock();
+ return completedFetches.stream().anyMatch(predicate);
+ } finally {
+ lock.unlock();
+ }
}
void add(CompletedFetch completedFetch) {
- completedFetches.add(completedFetch);
+ try {
+ lock.lock();
+ completedFetches.add(completedFetch);
+ notEmptyCondition.signalAll();
+ } finally {
+ lock.unlock();
+ }
}
void addAll(Collection<CompletedFetch> completedFetches) {
- this.completedFetches.addAll(completedFetches);
+ if (completedFetches == null || completedFetches.isEmpty())
+ return;
+
+ try {
+ lock.lock();
+ this.completedFetches.addAll(completedFetches);
+ notEmptyCondition.signalAll();
+ } finally {
+ lock.unlock();
+ }
}
CompletedFetch nextInLineFetch() {
- return nextInLineFetch;
+ try {
+ lock.lock();
+ return nextInLineFetch;
+ } finally {
+ lock.unlock();
+ }
}
- void setNextInLineFetch(CompletedFetch completedFetch) {
- this.nextInLineFetch = completedFetch;
+ void setNextInLineFetch(CompletedFetch nextInLineFetch) {
+ try {
+ lock.lock();
+ this.nextInLineFetch = nextInLineFetch;
+ } finally {
+ lock.unlock();
+ }
}
CompletedFetch peek() {
- return completedFetches.peek();
+ try {
+ lock.lock();
+ return completedFetches.peek();
+ } finally {
+ lock.unlock();
+ }
}
CompletedFetch poll() {
- return completedFetches.poll();
+ try {
+ lock.lock();
+ return completedFetches.poll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Allows the caller to await presence of data in the buffer. The method will block, returning only
+ * under one of the following conditions:
+ *
+ * <ol>
+ * <li>The buffer was already non-empty on entry</li>
+ * <li>The buffer was populated during the wait</li>
+ * <li>The remaining time on the {@link Timer timer} elapsed</li>
+ * <li>The thread was interrupted</li>
+ * </ol>
+ *
+ * @param timer Timer that provides time to wait
+ */
+ void awaitNotEmpty(Timer timer) {
+ try {
+ lock.lock();
+
+ while (isEmpty()) {
+ // Update the timer before we head into the loop in case it took a while to get the lock.
+ timer.update();
+
+ if (timer.isExpired())
+ break;
+
+ if (!notEmptyCondition.await(timer.remainingMs(), TimeUnit.MILLISECONDS)) {
+ break;
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptException("Timeout waiting for results from fetching records", e);
+ } finally {
+ lock.unlock();
+ timer.update();
+ }
}
/**
@@ -100,12 +192,22 @@ public class FetchBuffer implements Closeable {
* @param partitions {@link Set} of {@link TopicPartition}s for which any buffered data should be kept
*/
void retainAll(final Set<TopicPartition> partitions) {
- completedFetches.removeIf(cf -> maybeDrain(partitions, cf));
+ try {
+ lock.lock();
+
+ completedFetches.removeIf(cf -> maybeDrain(partitions, cf));
- if (maybeDrain(partitions, nextInLineFetch))
- nextInLineFetch = null;
+ if (maybeDrain(partitions, nextInLineFetch))
+ nextInLineFetch = null;
+ } finally {
+ lock.unlock();
+ }
}
+ /**
+ * Drains (i.e. <em>removes</em>) the contents of the given {@link CompletedFetch} as its data should not
+ * be returned to the user.
+ */
private boolean maybeDrain(final Set<TopicPartition> partitions, final CompletedFetch completedFetch) {
if (completedFetch != null && !partitions.contains(completedFetch.partition)) {
log.debug("Removing {} from buffered fetch data as it is not in the set of partitions to retain ({})", completedFetch.partition, partitions);
@@ -122,28 +224,33 @@ public class FetchBuffer implements Closeable {
* @return {@link TopicPartition Partition} set
*/
Set<TopicPartition> bufferedPartitions() {
- final Set<TopicPartition> partitions = new HashSet<>();
+ try {
+ lock.lock();
- if (nextInLineFetch != null && !nextInLineFetch.isConsumed()) {
- partitions.add(nextInLineFetch.partition);
- }
+ final Set<TopicPartition> partitions = new HashSet<>();
+
+ if (nextInLineFetch != null && !nextInLineFetch.isConsumed()) {
+ partitions.add(nextInLineFetch.partition);
+ }
- completedFetches.forEach(cf -> partitions.add(cf.partition));
- return partitions;
+ completedFetches.forEach(cf -> partitions.add(cf.partition));
+ return partitions;
+ } finally {
+ lock.unlock();
+ }
}
@Override
public void close() {
- idempotentCloser.close(() -> {
- log.debug("Closing the fetch buffer");
+ try {
+ lock.lock();
- if (nextInLineFetch != null) {
- nextInLineFetch.drain();
- nextInLineFetch = null;
- }
-
- completedFetches.forEach(CompletedFetch::drain);
- completedFetches.clear();
- }, () -> log.warn("The fetch buffer was previously closed"));
+ idempotentCloser.close(
+ () -> retainAll(Collections.emptySet()),
+ () -> log.warn("The fetch buffer was already closed")
+ );
+ } finally {
+ lock.unlock();
+ }
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java
index 0e1f2f18a3f..e98441321d3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java
@@ -54,20 +54,23 @@ public class FetchCollector<K, V> {
private final Logger log;
private final ConsumerMetadata metadata;
private final SubscriptionState subscriptions;
- private final FetchConfig<K, V> fetchConfig;
+ private final FetchConfig fetchConfig;
+ private final Deserializers<K, V> deserializers;
private final FetchMetricsManager metricsManager;
private final Time time;
public FetchCollector(final LogContext logContext,
final ConsumerMetadata metadata,
final SubscriptionState subscriptions,
- final FetchConfig<K, V> fetchConfig,
+ final FetchConfig fetchConfig,
+ final Deserializers<K, V> deserializers,
final FetchMetricsManager metricsManager,
final Time time) {
this.log = logContext.logger(FetchCollector.class);
this.metadata = metadata;
this.subscriptions = subscriptions;
this.fetchConfig = fetchConfig;
+ this.deserializers = deserializers;
this.metricsManager = metricsManager;
this.time = time;
}
@@ -162,7 +165,9 @@ public class FetchCollector<K, V> {
throw new IllegalStateException("Missing position for fetchable partition " + tp);
if (nextInLineFetch.nextFetchOffset() == position.offset) {
- List<ConsumerRecord<K, V>> partRecords = nextInLineFetch.fetchRecords(fetchConfig, fetchConfig.maxPollRecords);
+ List<ConsumerRecord<K, V>> partRecords = nextInLineFetch.fetchRecords(fetchConfig,
+ deserializers,
+ fetchConfig.maxPollRecords);
log.trace("Returning {} fetched records at offset {} for assigned partition {}",
partRecords.size(), position, tp);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java
index 66670b54472..24de49fee1d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java
@@ -19,57 +19,30 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.IsolationLevel;
-import org.apache.kafka.common.serialization.Deserializer;
-import java.util.Objects;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel;
/**
* {@link FetchConfig} represents the static configuration for fetching records from Kafka. It is simply a way
* to bundle the immutable settings that were presented at the time the {@link Consumer} was created for later use by
* classes like {@link Fetcher}, {@link CompletedFetch}, etc.
- *
- * <p/>
- *
- * In most cases, the values stored and returned by {@link FetchConfig} will be those stored in the following
- * {@link ConsumerConfig consumer configuration} settings:
- *
- * <ul>
- * <li>{@link #minBytes}: {@link ConsumerConfig#FETCH_MIN_BYTES_CONFIG}</li>
- * <li>{@link #maxBytes}: {@link ConsumerConfig#FETCH_MAX_BYTES_CONFIG}</li>
- * <li>{@link #maxWaitMs}: {@link ConsumerConfig#FETCH_MAX_WAIT_MS_CONFIG}</li>
- * <li>{@link #fetchSize}: {@link ConsumerConfig#MAX_PARTITION_FETCH_BYTES_CONFIG}</li>
- * <li>{@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}</li>
- * <li>{@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}</li>
- * <li>{@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}</li>
- * <li>{@link #deserializers}:
- * {@link ConsumerConfig#KEY_DESERIALIZER_CLASS_CONFIG}/{@link ConsumerConfig#VALUE_DESERIALIZER_CLASS_CONFIG}
- * </li>
- * <li>{@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}</li>
- * </ul>
- *
- * However, there are places in the code where additional logic is used to determine these fetch-related configuration
- * values. In those cases, the values are calculated outside of this class and simply passed in when constructed.
- *
- * <p/>
- *
- * Note: the {@link Deserializer deserializers} used for the key and value are not closed by this class. They should be
- * closed by the creator of the {@link FetchConfig}.
- *
- * @param <K> Type used to {@link Deserializer deserialize} the message/record key
- * @param <V> Type used to {@link Deserializer deserialize} the message/record value
*/
-public class FetchConfig<K, V> {
+public class FetchConfig {
- final int minBytes;
- final int maxBytes;
- final int maxWaitMs;
- final int fetchSize;
- final int maxPollRecords;
- final boolean checkCrcs;
- final String clientRackId;
- final Deserializers<K, V> deserializers;
- final IsolationLevel isolationLevel;
+ public final int minBytes;
+ public final int maxBytes;
+ public final int maxWaitMs;
+ public final int fetchSize;
+ public final int maxPollRecords;
+ public final boolean checkCrcs;
+ public final String clientRackId;
+ public final IsolationLevel isolationLevel;
+ /**
+ * Constructs a new {@link FetchConfig} using explicitly provided values. This is provided here for tests that
+ * want to exercise different scenarios can construct specific configuration values rather than going through
+ * the hassle of constructing a {@link ConsumerConfig}.
+ */
public FetchConfig(int minBytes,
int maxBytes,
int maxWaitMs,
@@ -77,7 +50,6 @@ public class FetchConfig<K, V> {
int maxPollRecords,
boolean checkCrcs,
String clientRackId,
- Deserializers<K, V> deserializers,
IsolationLevel isolationLevel) {
this.minBytes = minBytes;
this.maxBytes = maxBytes;
@@ -86,13 +58,27 @@ public class FetchConfig<K, V> {
this.maxPollRecords = maxPollRecords;
this.checkCrcs = checkCrcs;
this.clientRackId = clientRackId;
- this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null");
this.isolationLevel = isolationLevel;
}
- public FetchConfig(ConsumerConfig config,
- Deserializers<K, V> deserializers,
- IsolationLevel isolationLevel) {
+ /**
+ * Constructs a new {@link FetchConfig} using values from the given {@link ConsumerConfig consumer configuration}
+ * settings:
+ *
+ * <ul>
+ * <li>{@link #minBytes}: {@link ConsumerConfig#FETCH_MIN_BYTES_CONFIG}</li>
+ * <li>{@link #maxBytes}: {@link ConsumerConfig#FETCH_MAX_BYTES_CONFIG}</li>
+ * <li>{@link #maxWaitMs}: {@link ConsumerConfig#FETCH_MAX_WAIT_MS_CONFIG}</li>
+ * <li>{@link #fetchSize}: {@link ConsumerConfig#MAX_PARTITION_FETCH_BYTES_CONFIG}</li>
+ * <li>{@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}</li>
+ * <li>{@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}</li>
+ * <li>{@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}</li>
+ * <li>{@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}</li>
+ * </ul>
+ *
+ * @param config Consumer configuration
+ */
+ public FetchConfig(ConsumerConfig config) {
this.minBytes = config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG);
this.maxBytes = config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG);
this.maxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG);
@@ -100,8 +86,7 @@ public class FetchConfig<K, V> {
this.maxPollRecords = config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG);
this.checkCrcs = config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG);
this.clientRackId = config.getString(ConsumerConfig.CLIENT_RACK_CONFIG);
- this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null");
- this.isolationLevel = isolationLevel;
+ this.isolationLevel = configuredIsolationLevel(config);
}
@Override
@@ -114,7 +99,6 @@ public class FetchConfig<K, V> {
", maxPollRecords=" + maxPollRecords +
", checkCrcs=" + checkCrcs +
", clientRackId='" + clientRackId + '\'' +
- ", deserializers=" + deserializers +
", isolationLevel=" + isolationLevel +
'}';
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java
new file mode 100644
index 00000000000..f98d7b3d9fa
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.FetchSessionHandler;
+import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult;
+import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.requests.FetchRequest;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Time;
+
+/**
+ * {@code FetchRequestManager} is responsible for generating {@link FetchRequest} that represent the
+ * {@link SubscriptionState#fetchablePartitions(Predicate)} based on the user's topic subscription/partition
+ * assignment.
+ */
+public class FetchRequestManager extends AbstractFetch implements RequestManager {
+
+ private final NetworkClientDelegate networkClientDelegate;
+
+ FetchRequestManager(final LogContext logContext,
+ final Time time,
+ final ConsumerMetadata metadata,
+ final SubscriptionState subscriptions,
+ final FetchConfig fetchConfig,
+ final FetchBuffer fetchBuffer,
+ final FetchMetricsManager metricsManager,
+ final NetworkClientDelegate networkClientDelegate) {
+ super(logContext, metadata, subscriptions, fetchConfig, fetchBuffer, metricsManager, time);
+ this.networkClientDelegate = networkClientDelegate;
+ }
+
+ @Override
+ protected boolean isUnavailable(Node node) {
+ return networkClientDelegate.isUnavailable(node);
+ }
+
+ @Override
+ protected void maybeThrowAuthFailure(Node node) {
+ networkClientDelegate.maybeThrowAuthFailure(node);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public PollResult poll(long currentTimeMs) {
+ return pollInternal(
+ prepareFetchRequests(),
+ this::handleFetchSuccess,
+ this::handleFetchFailure
+ );
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public PollResult pollOnClose() {
+ return pollInternal(
+ prepareCloseFetchSessionRequests(),
+ this::handleCloseFetchSessionSuccess,
+ this::handleCloseFetchSessionFailure
+ );
+ }
+
+ /**
+ * Creates the {@link PollResult poll result} that contains a list of zero or more
+ * {@link FetchRequest.Builder fetch requests}.
+ *
+ * @param fetchRequests {@link Map} of {@link Node nodes} to their {@link FetchSessionHandler.FetchRequestData}
+ * @param successHandler {@link ResponseHandler Handler for successful responses}
+ * @param errorHandler {@link ResponseHandler Handler for failure responses}
+ * @return {@link PollResult}
+ */
+ private PollResult pollInternal(Map<Node, FetchSessionHandler.FetchRequestData> fetchRequests,
+ ResponseHandler<ClientResponse> successHandler,
+ ResponseHandler<Throwable> errorHandler) {
+ List<UnsentRequest> requests = fetchRequests.entrySet().stream().map(entry -> {
+ final Node fetchTarget = entry.getKey();
+ final FetchSessionHandler.FetchRequestData data = entry.getValue();
+ final FetchRequest.Builder request = createFetchRequest(fetchTarget, data);
+ final BiConsumer<ClientResponse, Throwable> responseHandler = (clientResponse, error) -> {
+ if (error != null)
+ errorHandler.handle(fetchTarget, data, error);
+ else
+ successHandler.handle(fetchTarget, data, clientResponse);
+ };
+
+ return new UnsentRequest(request, fetchTarget, responseHandler);
+ }).collect(Collectors.toList());
+
+ return new PollResult(requests);
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index dec02c6b90f..05df99373e1 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -16,16 +16,23 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.FetchSessionHandler;
+import org.apache.kafka.clients.NetworkClient;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.IdempotentCloser;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Timer;
+import org.slf4j.Logger;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
/**
@@ -47,66 +54,147 @@ import java.util.Map;
* on a different thread.</li>
* </ul>
*/
-public class Fetcher<K, V> extends AbstractFetch<K, V> {
+public class Fetcher<K, V> extends AbstractFetch {
+ private final Logger log;
+ private final ConsumerNetworkClient client;
private final FetchCollector<K, V> fetchCollector;
public Fetcher(LogContext logContext,
ConsumerNetworkClient client,
ConsumerMetadata metadata,
SubscriptionState subscriptions,
- FetchConfig<K, V> fetchConfig,
+ FetchConfig fetchConfig,
+ Deserializers<K, V> deserializers,
FetchMetricsManager metricsManager,
Time time) {
- super(logContext, client, metadata, subscriptions, fetchConfig, metricsManager, time);
+ super(logContext, metadata, subscriptions, fetchConfig, new FetchBuffer(logContext), metricsManager, time);
+ this.log = logContext.logger(Fetcher.class);
+ this.client = client;
this.fetchCollector = new FetchCollector<>(logContext,
metadata,
subscriptions,
fetchConfig,
+ deserializers,
metricsManager,
time);
}
+ @Override
+ protected boolean isUnavailable(Node node) {
+ return client.isUnavailable(node);
+ }
+
+ @Override
+ protected void maybeThrowAuthFailure(Node node) {
+ client.maybeThrowAuthFailure(node);
+ }
+
public void clearBufferedDataForUnassignedPartitions(Collection<TopicPartition> assignedPartitions) {
fetchBuffer.retainAll(new HashSet<>(assignedPartitions));
}
/**
- * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have
+ * Set up a fetch request for any node that we have assigned partitions for which doesn't already have
* an in-flight fetch or pending fetch data.
* @return number of fetches sent
*/
public synchronized int sendFetches() {
- Map<Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareFetchRequests();
+ final Map<Node, FetchSessionHandler.FetchRequestData> fetchRequests = prepareFetchRequests();
+ sendFetchesInternal(
+ fetchRequests,
+ (fetchTarget, data, clientResponse) -> {
+ synchronized (Fetcher.this) {
+ handleFetchSuccess(fetchTarget, data, clientResponse);
+ }
+ },
+ (fetchTarget, data, error) -> {
+ synchronized (Fetcher.this) {
+ handleFetchFailure(fetchTarget, data, error);
+ }
+ });
+ return fetchRequests.size();
+ }
- for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequestMap.entrySet()) {
+ protected void maybeCloseFetchSessions(final Timer timer) {
+ final List<RequestFuture<ClientResponse>> requestFutures = sendFetchesInternal(
+ prepareCloseFetchSessionRequests(),
+ this::handleCloseFetchSessionSuccess,
+ this::handleCloseFetchSessionFailure
+ );
+
+ // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until
+ // all requests have received a response.
+ while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) {
+ client.poll(timer, null, true);
+ timer.update();
+ }
+
+ if (!requestFutures.stream().allMatch(RequestFuture::isDone)) {
+ // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown
+ // here.
+ log.debug("All requests couldn't be sent in the specific timeout period {}ms. " +
+ "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " +
+ "KafkaConsumer.close(Duration timeout)", timer.timeoutMs());
+ }
+ }
+
+ public Fetch<K, V> collectFetch() {
+ return fetchCollector.collectFetch(fetchBuffer);
+ }
+
+ /**
+ * This method is called by {@link #close(Timer)} which is guarded by the {@link IdempotentCloser}) such as to only
+ * be executed once the first time that any of the {@link #close()} methods are called. Subclasses can override
+ * this method without the need for extra synchronization at the instance level.
+ *
+ * @param timer Timer to enforce time limit
+ */
+ // Visible for testing
+ protected void closeInternal(Timer timer) {
+ // we do not need to re-enable wake-ups since we are closing already
+ client.disableWakeups();
+ maybeCloseFetchSessions(timer);
+ super.closeInternal(timer);
+ }
+
+ /**
+ * Creates the {@link FetchRequest.Builder fetch request},
+ * {@link NetworkClient#send(ClientRequest, long) enqueues/sends it, and adds the {@link RequestFuture callback}
+ * for the response.
+ *
+ * @param fetchRequests {@link Map} of {@link Node nodes} to their
+ * {@link FetchSessionHandler.FetchRequestData request data}
+ * @param successHandler {@link ResponseHandler Handler for successful responses}
+ * @param errorHandler {@link ResponseHandler Handler for failure responses}
+ * @return List of {@link RequestFuture callbacks}
+ */
+ private List<RequestFuture<ClientResponse>> sendFetchesInternal(Map<Node, FetchSessionHandler.FetchRequestData> fetchRequests,
+ ResponseHandler<ClientResponse> successHandler,
+ ResponseHandler<Throwable> errorHandler) {
+ final List<RequestFuture<ClientResponse>> requestFutures = new ArrayList<>();
+
+ for (Map.Entry<Node, FetchSessionHandler.FetchRequestData> entry : fetchRequests.entrySet()) {
final Node fetchTarget = entry.getKey();
final FetchSessionHandler.FetchRequestData data = entry.getValue();
final FetchRequest.Builder request = createFetchRequest(fetchTarget, data);
- RequestFutureListener<ClientResponse> listener = new RequestFutureListener<ClientResponse>() {
+ final RequestFuture<ClientResponse> responseFuture = client.send(fetchTarget, request);
+
+ responseFuture.addListener(new RequestFutureListener<ClientResponse>() {
@Override
public void onSuccess(ClientResponse resp) {
- synchronized (Fetcher.this) {
- handleFetchResponse(fetchTarget, data, resp);
- }
+ successHandler.handle(fetchTarget, data, resp);
}
@Override
public void onFailure(RuntimeException e) {
- synchronized (Fetcher.this) {
- handleFetchResponse(fetchTarget, e);
- }
+ errorHandler.handle(fetchTarget, data, e);
}
- };
+ });
- final RequestFuture<ClientResponse> future = client.send(fetchTarget, request);
- future.addListener(listener);
+ requestFutures.add(responseFuture);
}
- return fetchRequestMap.size();
- }
-
- public Fetch<K, V> collectFetch() {
- return fetchCollector.collectFetch(fetchBuffer);
+ return requestFutures;
}
}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
index 2e2ed9f2ab4..2172313054d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java
@@ -18,6 +18,8 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
import org.apache.kafka.common.errors.GroupAuthorizationException;
import org.apache.kafka.common.errors.RetriableException;
import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData;
@@ -58,6 +60,7 @@ import java.util.Collections;
* <p>See {@link HeartbeatRequestState} for more details.</p>
*/
public class HeartbeatRequestManager implements RequestManager {
+
private final Logger logger;
private final Time time;
@@ -90,22 +93,22 @@ public class HeartbeatRequestManager implements RequestManager {
/**
* ErrorEventHandler allows the background thread to propagate errors back to the user
*/
- private final ErrorEventHandler nonRetriableErrorHandler;
+ private final BackgroundEventHandler backgroundEventHandler;
public HeartbeatRequestManager(
- final Time time,
final LogContext logContext,
+ final Time time,
final ConsumerConfig config,
final CoordinatorRequestManager coordinatorRequestManager,
final SubscriptionState subscriptions,
final MembershipManager membershipManager,
- final ErrorEventHandler nonRetriableErrorHandler) {
+ final BackgroundEventHandler backgroundEventHandler) {
this.coordinatorRequestManager = coordinatorRequestManager;
this.time = time;
this.logger = logContext.logger(getClass());
this.subscriptions = subscriptions;
this.membershipManager = membershipManager;
- this.nonRetriableErrorHandler = nonRetriableErrorHandler;
+ this.backgroundEventHandler = backgroundEventHandler;
this.rebalanceTimeoutMs = config.getInt(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG);
long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG);
@@ -122,7 +125,7 @@ public class HeartbeatRequestManager implements RequestManager {
final SubscriptionState subscriptions,
final MembershipManager membershipManager,
final HeartbeatRequestState heartbeatRequestState,
- final ErrorEventHandler nonRetriableErrorHandler) {
+ final BackgroundEventHandler backgroundEventHandler) {
this.logger = logContext.logger(this.getClass());
this.time = time;
this.subscriptions = subscriptions;
@@ -130,7 +133,7 @@ public class HeartbeatRequestManager implements RequestManager {
this.coordinatorRequestManager = coordinatorRequestManager;
this.heartbeatRequestState = heartbeatRequestState;
this.membershipManager = membershipManager;
- this.nonRetriableErrorHandler = nonRetriableErrorHandler;
+ this.backgroundEventHandler = backgroundEventHandler;
}
/**
@@ -146,18 +149,14 @@ public class HeartbeatRequestManager implements RequestManager {
*/
@Override
public NetworkClientDelegate.PollResult poll(long currentTimeMs) {
- if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) {
- return new NetworkClientDelegate.PollResult(
- Long.MAX_VALUE, Collections.emptyList());
- }
+ if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat())
+ return NetworkClientDelegate.PollResult.EMPTY;
// TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be
// implemented either with or after the partition reconciliation logic.
- if (!heartbeatRequestState.canSendRequest(currentTimeMs)) {
- return new NetworkClientDelegate.PollResult(
- heartbeatRequestState.nextHeartbeatMs(currentTimeMs),
- Collections.emptyList());
- }
+ if (!heartbeatRequestState.canSendRequest(currentTimeMs))
+ return new NetworkClientDelegate.PollResult(heartbeatRequestState.nextHeartbeatMs(currentTimeMs));
+
this.heartbeatRequestState.onSendAttempt(currentTimeMs);
NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest();
return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request));
@@ -198,7 +197,7 @@ public class HeartbeatRequestManager implements RequestManager {
this.heartbeatRequestState.onFailedAttempt(responseTimeMs);
if (exception instanceof RetriableException) {
String message = String.format("GroupHeartbeatRequest failed because of the retriable exception. " +
- "Will retry in %s ms: {}",
+ "Will retry in %s ms: %s",
heartbeatRequestState.remainingBackoffMs(responseTimeMs),
exception.getMessage());
logger.debug(message);
@@ -223,12 +222,13 @@ public class HeartbeatRequestManager implements RequestManager {
final long currentTimeMs) {
Errors error = Errors.forCode(response.data().errorCode());
String errorMessage = response.data().errorMessage();
+ String message;
// TODO: upon encountering a fatal/fenced error, trigger onPartitionLost logic to give up the current
// assignments.
switch (error) {
case NOT_COORDINATOR:
// the manager should retry immediately when the coordinator node becomes available again
- String message = String.format("GroupHeartbeatRequest failed because the group coordinator %s is incorrect. " +
+ message = String.format("GroupHeartbeatRequest failed because the group coordinator %s is incorrect. " +
"Will attempt to find the coordinator again and retry",
coordinatorRequestManager.coordinator());
logInfo(message, response, currentTimeMs);
@@ -274,7 +274,7 @@ public class HeartbeatRequestManager implements RequestManager {
break;
case FENCED_MEMBER_EPOCH:
- message = String.format("GroupHeartbeatRequest failed because member epoch %s is invalid. " +
+ message = String.format("GroupHeartbeatRequest failed because member ID %s with epoch %s is invalid. " +
"Will abandon all partitions and rejoin the group",
membershipManager.memberId(), membershipManager.memberEpoch());
logInfo(message, response, currentTimeMs);
@@ -282,7 +282,7 @@ public class HeartbeatRequestManager implements RequestManager {
break;
case UNKNOWN_MEMBER_ID:
- message = String.format("GroupHeartbeatRequest failed because member id %s is invalid. " +
+ message = String.format("GroupHeartbeatRequest failed because member of unknown ID %s with epoch %s is invalid. " +
"Will abandon all partitions and rejoin the group",
membershipManager.memberId(), membershipManager.memberEpoch());
logInfo(message, response, currentTimeMs);
@@ -307,7 +307,7 @@ public class HeartbeatRequestManager implements RequestManager {
}
private void handleFatalFailure(Throwable error) {
- nonRetriableErrorHandler.handle(error);
+ backgroundEventHandler.add(new ErrorBackgroundEvent(error));
membershipManager.transitionToFailed();
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java
index 2e61b0788f8..47485ed46ca 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java
@@ -16,8 +16,10 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.ClientUtils;
import org.apache.kafka.clients.KafkaClient;
import org.apache.kafka.clients.NetworkClientUtils;
import org.apache.kafka.clients.RequestCompletionHandler;
@@ -26,6 +28,7 @@ import org.apache.kafka.common.Node;
import org.apache.kafka.common.errors.AuthenticationException;
import org.apache.kafka.common.errors.DisconnectException;
import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
@@ -35,27 +38,29 @@ import org.slf4j.Logger;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Queue;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX;
/**
* A wrapper around the {@link org.apache.kafka.clients.NetworkClient} to handle network poll and send operations.
*/
public class NetworkClientDelegate implements AutoCloseable {
+
private final KafkaClient client;
private final Time time;
private final Logger log;
private final int requestTimeoutMs;
private final Queue<UnsentRequest> unsentRequests;
private final long retryBackoffMs;
- private final Set<Node> tryConnectNodes;
public NetworkClientDelegate(
final Time time,
@@ -68,9 +73,40 @@ public class NetworkClientDelegate implements AutoCloseable {
this.unsentRequests = new ArrayDeque<>();
this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG);
this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
- this.tryConnectNodes = new HashSet<>();
}
+ // Visible for testing
+ Queue<UnsentRequest> unsentRequests() {
+ return unsentRequests;
+ }
+
+ /**
+ * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in
+ * reconnect backoff window following the disconnect).
+ *
+ * @param node {@link Node} to check for availability
+ * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time)
+ */
+ public boolean isUnavailable(Node node) {
+ return NetworkClientUtils.isUnavailable(client, node, time);
+ }
+
+ /**
+ * Checks for an authentication error on a given node and throws the exception if it exists.
+ *
+ * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown
+ * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node)
+ */
+ public void maybeThrowAuthFailure(Node node) {
+ NetworkClientUtils.maybeThrowAuthFailure(client, node);
+ }
+
+ /**
+ * Initiate a connection if currently possible. This is only really useful for resetting
+ * the failed status of a socket.
+ *
+ * @param node The node to connect to
+ */
public void tryConnect(Node node) {
NetworkClientUtils.tryConnect(client, node, time);
}
@@ -81,7 +117,6 @@ public class NetworkClientDelegate implements AutoCloseable {
*
* @param timeoutMs timeout time
* @param currentTimeMs current time
- * @return a list of client response
*/
public void poll(final long timeoutMs, final long currentTimeMs) {
trySend(currentTimeMs);
@@ -119,8 +154,7 @@ public class NetworkClientDelegate implements AutoCloseable {
}
}
- private boolean doSend(final UnsentRequest r,
- final long currentTimeMs) {
+ boolean doSend(final UnsentRequest r, final long currentTimeMs) {
Node node = r.node.orElse(client.leastLoadedNode(currentTimeMs));
if (node == null || nodeUnavailable(node)) {
log.debug("No broker available to send the request: {}. Retrying.", r);
@@ -137,7 +171,7 @@ public class NetworkClientDelegate implements AutoCloseable {
return true;
}
- private void checkDisconnects(final long currentTimeMs) {
+ protected void checkDisconnects(final long currentTimeMs) {
// Check the connection of the unsent request. Disconnect the disconnected node if it is unable to be connected.
Iterator<UnsentRequest> iter = unsentRequests.iterator();
while (iter.hasNext()) {
@@ -190,26 +224,47 @@ public class NetworkClientDelegate implements AutoCloseable {
this.client.close();
}
+ public long addAll(PollResult pollResult) {
+ addAll(pollResult.unsentRequests);
+ return pollResult.timeUntilNextPollMs;
+ }
+
public void addAll(final List<UnsentRequest> requests) {
- requests.forEach(u -> {
- u.setTimer(this.time, this.requestTimeoutMs);
- });
- this.unsentRequests.addAll(requests);
+ if (!requests.isEmpty()) {
+ requests.forEach(ur -> ur.setTimer(time, requestTimeoutMs));
+ unsentRequests.addAll(requests);
+ }
}
public static class PollResult {
+
+ public static final long WAIT_FOREVER = Long.MAX_VALUE;
+ public static final PollResult EMPTY = new PollResult(WAIT_FOREVER);
public final long timeUntilNextPollMs;
public final List<UnsentRequest> unsentRequests;
- public PollResult(final long timeMsTillNextPoll, final List<UnsentRequest> unsentRequests) {
- this.timeUntilNextPollMs = timeMsTillNextPoll;
+ public PollResult(final long timeUntilNextPollMs, final List<UnsentRequest> unsentRequests) {
+ this.timeUntilNextPollMs = timeUntilNextPollMs;
this.unsentRequests = Collections.unmodifiableList(unsentRequests);
}
+
+ public PollResult(final List<UnsentRequest> unsentRequests) {
+ this(WAIT_FOREVER, unsentRequests);
+ }
+
+ public PollResult(final UnsentRequest unsentRequest) {
+ this(Collections.singletonList(unsentRequest));
+ }
+
+ public PollResult(final long timeUntilNextPollMs) {
+ this(timeUntilNextPollMs, Collections.emptyList());
+ }
}
+
public static class UnsentRequest {
private final AbstractRequest.Builder<?> requestBuilder;
private final FutureCompletionHandler handler;
- private Optional<Node> node; // empty if random node can be chosen
+ private final Optional<Node> node; // empty if random node can be chosen
private Timer timer;
public UnsentRequest(final AbstractRequest.Builder<?> requestBuilder,
@@ -227,6 +282,12 @@ public class NetworkClientDelegate implements AutoCloseable {
this.handler.future().whenComplete(callback);
}
+ public UnsentRequest(final AbstractRequest.Builder<?> requestBuilder,
+ final Node node,
+ final BiConsumer<ClientResponse, Throwable> callback) {
+ this(requestBuilder, Optional.of(node), callback);
+ }
+
public void setTimer(final Time time, final long requestTimeoutMs) {
this.timer = time.timer(requestTimeoutMs);
}
@@ -243,6 +304,10 @@ public class NetworkClientDelegate implements AutoCloseable {
return requestBuilder;
}
+ Optional<Node> node() {
+ return node;
+ }
+
@Override
public String toString() {
return "UnsentRequest{" +
@@ -291,4 +356,32 @@ public class NetworkClientDelegate implements AutoCloseable {
return future;
}
}
+
+ /**
+ * Creates a {@link Supplier} for deferred creation during invocation by
+ * {@link ConsumerNetworkThread}.
+ */
+ public static Supplier<NetworkClientDelegate> supplier(final Time time,
+ final LogContext logContext,
+ final ConsumerMetadata metadata,
+ final ConsumerConfig config,
+ final ApiVersions apiVersions,
+ final Metrics metrics,
+ final FetchMetricsManager fetchMetricsManager) {
+ return new CachedSupplier<NetworkClientDelegate>() {
+ @Override
+ protected NetworkClientDelegate create() {
+ KafkaClient client = ClientUtils.createNetworkClient(config,
+ metrics,
+ CONSUMER_METRIC_GROUP_PREFIX,
+ logContext,
+ apiVersions,
+ time,
+ CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION,
+ metadata,
+ fetchMetricsManager.throttleTimeSensor());
+ return new NetworkClientDelegate(time, config, logContext, client);
+ }
+ };
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java
index b7fdefeb0d1..9239811f7d6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java
@@ -19,6 +19,7 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.NodeApiVersions;
import org.apache.kafka.clients.consumer.LogTruncationException;
+import org.apache.kafka.clients.consumer.NoOffsetForPartitionException;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.clients.consumer.OffsetResetStrategy;
@@ -262,7 +263,7 @@ class OffsetFetcherUtils {
else if (strategy == OffsetResetStrategy.LATEST)
return ListOffsetsRequest.LATEST_TIMESTAMP;
else
- return null;
+ throw new NoOffsetForPartitionException(partition);
}
static Set<String> topicsForPartitions(Collection<TopicPartition> partitions) {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java
index a1cb8d54a97..700e2ab6e17 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java
@@ -24,6 +24,8 @@ import org.apache.kafka.clients.consumer.LogTruncationException;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.clients.consumer.internals.OffsetFetcherUtils.ListOffsetData;
import org.apache.kafka.clients.consumer.internals.OffsetFetcherUtils.ListOffsetResult;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
import org.apache.kafka.common.ClusterResource;
import org.apache.kafka.common.ClusterResourceListener;
import org.apache.kafka.common.IsolationLevel;
@@ -83,6 +85,7 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
private final Time time;
private final ApiVersions apiVersions;
private final NetworkClientDelegate networkClientDelegate;
+ private final BackgroundEventHandler backgroundEventHandler;
@SuppressWarnings("this-escape")
public OffsetsRequestManager(final SubscriptionState subscriptionState,
@@ -93,6 +96,7 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
final long requestTimeoutMs,
final ApiVersions apiVersions,
final NetworkClientDelegate networkClientDelegate,
+ final BackgroundEventHandler backgroundEventHandler,
final LogContext logContext) {
requireNonNull(subscriptionState);
requireNonNull(metadata);
@@ -100,6 +104,7 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
requireNonNull(time);
requireNonNull(apiVersions);
requireNonNull(networkClientDelegate);
+ requireNonNull(backgroundEventHandler);
requireNonNull(logContext);
this.metadata = metadata;
@@ -112,11 +117,12 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
this.requestTimeoutMs = requestTimeoutMs;
this.apiVersions = apiVersions;
this.networkClientDelegate = networkClientDelegate;
+ this.backgroundEventHandler = backgroundEventHandler;
this.offsetFetcherUtils = new OffsetFetcherUtils(logContext, metadata, subscriptionState,
time, retryBackoffMs, apiVersions);
// Register the cluster metadata update callback. Note this only relies on the
// requestsToRetry initialized above, and won't be invoked until all managers are
- // initialized and the background thread started.
+ // initialized and the network thread started.
this.metadata.addClusterUpdateListener(this);
}
@@ -127,10 +133,10 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
*/
@Override
public NetworkClientDelegate.PollResult poll(final long currentTimeMs) {
- NetworkClientDelegate.PollResult pollResult =
- new NetworkClientDelegate.PollResult(Long.MAX_VALUE, new ArrayList<>(requestsToSend));
- this.requestsToSend.clear();
- return pollResult;
+ // Copy the outgoing request list and clear it.
+ List<NetworkClientDelegate.UnsentRequest> unsentRequests = new ArrayList<>(requestsToSend);
+ requestsToSend.clear();
+ return new NetworkClientDelegate.PollResult(unsentRequests);
}
/**
@@ -188,7 +194,14 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis
* this function (ex. {@link org.apache.kafka.common.errors.TopicAuthorizationException})
*/
public void resetPositionsIfNeeded() {
- Map<TopicPartition, Long> offsetResetTimestamps = offsetFetcherUtils.getOffsetResetTimestamp();
+ Map<TopicPartition, Long> offsetResetTimestamps;
+
+ try {
+ offsetResetTimestamps = offsetFetcherUtils.getOffsetResetTimestamp();
+ } catch (Exception e) {
+ backgroundEventHandler.add(new ErrorBackgroundEvent(e));
+ return;
+ }
if (offsetResetTimestamps.isEmpty())
return;
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java
index 195bee9ddd4..949616daa85 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java
@@ -20,25 +20,33 @@ import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.ClientUtils;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.GroupRebalanceConfig;
+import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
import org.apache.kafka.clients.consumer.ConsumerInterceptor;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.NoOffsetForPartitionException;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
import org.apache.kafka.clients.consumer.OffsetCommitCallback;
+import org.apache.kafka.clients.consumer.OffsetResetStrategy;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent;
import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler;
import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.EventHandler;
import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent;
import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent;
import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent;
import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent;
import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Metric;
import org.apache.kafka.common.MetricName;
@@ -48,11 +56,12 @@ import org.apache.kafka.common.Uuid;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.errors.InterruptException;
import org.apache.kafka.common.errors.InvalidGroupIdException;
-import org.apache.kafka.common.errors.WakeupException;
+import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.internals.ClusterResourceListeners;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.requests.ListOffsetsRequest;
import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.utils.AppInfoParser;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Timer;
@@ -67,121 +76,246 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Properties;
import java.util.Set;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.refreshCommittedOffsets;
import static org.apache.kafka.common.utils.Utils.closeQuietly;
import static org.apache.kafka.common.utils.Utils.isBlank;
import static org.apache.kafka.common.utils.Utils.join;
import static org.apache.kafka.common.utils.Utils.propsToMap;
/**
- * This prototype consumer uses the EventHandler to process application
- * events so that the network IO can be processed in a background thread. Visit
- * <a href="https://cwiki.apache.org/confluence/display/KAFKA/Proposal%3A+Consumer+Threading+Model+Refactor" >this document</a>
+ * This prototype consumer uses an {@link ApplicationEventHandler event handler} to process
+ * {@link ApplicationEvent application events} so that the network IO can be processed in a dedicated
+ * {@link ConsumerNetworkThread network thread}. Visit
+ * <a href="https://cwiki.apache.org/confluence/display/KAFKA/Proposal%3A+Consumer+Threading+Model+Refactor">this document</a>
* for detail implementation.
*/
public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
- static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000;
- private final LogContext logContext;
- private final EventHandler eventHandler;
+ private final ApplicationEventHandler applicationEventHandler;
private final Time time;
private final Optional<String> groupId;
- private final Logger log;
+ private final KafkaConsumerMetrics kafkaConsumerMetrics;
+ private Logger log;
+ private final String clientId;
+ private final BackgroundEventProcessor backgroundEventProcessor;
private final Deserializers<K, V> deserializers;
+
+ /**
+ * A thread-safe {@link FetchBuffer fetch buffer} for the results that are populated in the
+ * {@link ConsumerNetworkThread network thread} when the results are available. Because of the interaction
+ * of the fetch buffer in the application thread and the network I/O thread, this is shared between the
+ * two threads and is thus designed to be thread-safe.
+ */
+ private final FetchBuffer fetchBuffer;
+ private final FetchCollector<K, V> fetchCollector;
+ private final ConsumerInterceptors<K, V> interceptors;
+ private final IsolationLevel isolationLevel;
+
private final SubscriptionState subscriptions;
private final ConsumerMetadata metadata;
private final Metrics metrics;
+ private final long retryBackoffMs;
private final long defaultApiTimeoutMs;
+ private volatile boolean closed = false;
+ private final List<ConsumerPartitionAssignor> assignors;
+
+ // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates
+ private boolean cachedSubscriptionHasAllFetchPositions;
+ private final WakeupTrigger wakeupTrigger = new WakeupTrigger();
- private WakeupTrigger wakeupTrigger = new WakeupTrigger();
- public PrototypeAsyncConsumer(Properties properties,
- Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer) {
+ public PrototypeAsyncConsumer(final Properties properties,
+ final Deserializer<K> keyDeserializer,
+ final Deserializer<V> valueDeserializer) {
this(propsToMap(properties), keyDeserializer, valueDeserializer);
}
public PrototypeAsyncConsumer(final Map<String, Object> configs,
- final Deserializer<K> keyDeser,
- final Deserializer<V> valDeser) {
- this(new ConsumerConfig(appendDeserializerToConfig(configs, keyDeser, valDeser)), keyDeser, valDeser);
+ final Deserializer<K> keyDeserializer,
+ final Deserializer<V> valueDeserializer) {
+ this(new ConsumerConfig(appendDeserializerToConfig(configs, keyDeserializer, valueDeserializer)),
+ keyDeserializer,
+ valueDeserializer);
}
public PrototypeAsyncConsumer(final ConsumerConfig config,
final Deserializer<K> keyDeserializer,
final Deserializer<V> valueDeserializer) {
- this.time = Time.SYSTEM;
- GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config,
- GroupRebalanceConfig.ProtocolType.CONSUMER);
- this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId);
- this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG);
- this.logContext = createLogContext(config, groupRebalanceConfig);
- this.log = logContext.logger(getClass());
- this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer);
- this.subscriptions = createSubscriptionState(config, logContext);
- this.metrics = createMetrics(config, time);
- List<ConsumerInterceptor<K, V>> interceptorList = configuredConsumerInterceptors(config);
- ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners(
- metrics.reporters(),
- interceptorList,
- Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer));
- this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners);
- final List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config);
- metadata.bootstrap(addresses);
- this.eventHandler = new DefaultEventHandler(
- config,
- groupRebalanceConfig,
- logContext,
- subscriptions,
- new ApiVersions(),
- this.metrics,
- clusterResourceListeners,
- null // this is coming from the fetcher, but we don't have one
- );
+ this(Time.SYSTEM, config, keyDeserializer, valueDeserializer);
}
- // Visible for testing
- PrototypeAsyncConsumer(Time time,
- LogContext logContext,
- ConsumerConfig config,
- SubscriptionState subscriptions,
- ConsumerMetadata metadata,
- EventHandler eventHandler,
- Metrics metrics,
- Optional<String> groupId,
- int defaultApiTimeoutMs) {
- this.time = time;
- this.logContext = logContext;
+ public PrototypeAsyncConsumer(final Time time,
+ final ConsumerConfig config,
+ final Deserializer<K> keyDeserializer,
+ final Deserializer<V> valueDeserializer) {
+ try {
+ GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config,
+ GroupRebalanceConfig.ProtocolType.CONSUMER);
+
+ this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId);
+ this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG);
+ LogContext logContext = createLogContext(config, groupRebalanceConfig);
+ this.log = logContext.logger(getClass());
+ groupId.ifPresent(groupIdStr -> {
+ if (groupIdStr.isEmpty()) {
+ log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release.");
+ }
+ });
+
+ log.debug("Initializing the Kafka consumer");
+ this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG);
+ this.time = time;
+ this.metrics = createMetrics(config, time);
+ this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
+
+ List<ConsumerInterceptor<K, V>> interceptorList = configuredConsumerInterceptors(config);
+ this.interceptors = new ConsumerInterceptors<>(interceptorList);
+ this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer);
+ this.subscriptions = createSubscriptionState(config, logContext);
+ ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners(metrics.reporters(),
+ interceptorList,
+ Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer));
+ this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners);
+ final List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config);
+ metadata.bootstrap(addresses);
+
+ FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics);
+ FetchConfig fetchConfig = new FetchConfig(config);
+ this.isolationLevel = fetchConfig.isolationLevel;
+
+ ApiVersions apiVersions = new ApiVersions();
+ final BlockingQueue<ApplicationEvent> applicationEventQueue = new LinkedBlockingQueue<>();
+ final BlockingQueue<BackgroundEvent> backgroundEventQueue = new LinkedBlockingQueue<>();
+
+ // This FetchBuffer is shared between the application and network threads.
+ this.fetchBuffer = new FetchBuffer(logContext);
+ final Supplier<NetworkClientDelegate> networkClientDelegateSupplier = NetworkClientDelegate.supplier(time,
+ logContext,
+ metadata,
+ config,
+ apiVersions,
+ metrics,
+ fetchMetricsManager);
+ final Supplier<RequestManagers> requestManagersSupplier = RequestManagers.supplier(time,
+ logContext,
+ backgroundEventQueue,
+ metadata,
+ subscriptions,
+ fetchBuffer,
+ config,
+ groupRebalanceConfig,
+ apiVersions,
+ fetchMetricsManager,
+ networkClientDelegateSupplier);
+ final Supplier<ApplicationEventProcessor> applicationEventProcessorSupplier = ApplicationEventProcessor.supplier(logContext,
+ metadata,
+ applicationEventQueue,
+ requestManagersSupplier);
+ this.applicationEventHandler = new ApplicationEventHandler(logContext,
+ time,
+ applicationEventQueue,
+ applicationEventProcessorSupplier,
+ networkClientDelegateSupplier,
+ requestManagersSupplier);
+ this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue);
+ this.assignors = ConsumerPartitionAssignor.getAssignorInstances(
+ config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG),
+ config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId))
+ );
+
+ // no coordinator will be constructed for the default (null) group id
+ if (!groupId.isPresent()) {
+ config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG);
+ //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED);
+ }
+
+ // The FetchCollector is only used on the application thread.
+ this.fetchCollector = new FetchCollector<>(logContext,
+ metadata,
+ subscriptions,
+ fetchConfig,
+ deserializers,
+ fetchMetricsManager,
+ time);
+
+ this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX);
+
+ config.logUnused();
+ AppInfoParser.registerAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics, time.milliseconds());
+ log.debug("Kafka consumer initialized");
+ } catch (Throwable t) {
+ // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121
+ // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized.
+ if (this.log != null) {
+ close(Duration.ZERO, true);
+ }
+ // now propagate the exception
+ throw new KafkaException("Failed to construct kafka consumer", t);
+ }
+ }
+
+ public PrototypeAsyncConsumer(LogContext logContext,
+ String clientId,
+ Deserializers<K, V> deserializers,
+ FetchBuffer fetchBuffer,
+ FetchCollector<K, V> fetchCollector,
+ ConsumerInterceptors<K, V> interceptors,
+ Time time,
+ ApplicationEventHandler applicationEventHandler,
+ BlockingQueue<BackgroundEvent> backgroundEventQueue,
+ Metrics metrics,
+ SubscriptionState subscriptions,
+ ConsumerMetadata metadata,
+ long retryBackoffMs,
+ int defaultApiTimeoutMs,
+ List<ConsumerPartitionAssignor> assignors,
+ String groupId) {
this.log = logContext.logger(getClass());
this.subscriptions = subscriptions;
- this.metadata = metadata;
+ this.clientId = clientId;
+ this.fetchBuffer = fetchBuffer;
+ this.fetchCollector = fetchCollector;
+ this.isolationLevel = IsolationLevel.READ_UNCOMMITTED;
+ this.interceptors = Objects.requireNonNull(interceptors);
+ this.time = time;
+ this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue);
this.metrics = metrics;
- this.groupId = groupId;
+ this.groupId = Optional.ofNullable(groupId);
+ this.metadata = metadata;
+ this.retryBackoffMs = retryBackoffMs;
this.defaultApiTimeoutMs = defaultApiTimeoutMs;
- this.deserializers = new Deserializers<>(config);
- this.eventHandler = eventHandler;
+ this.deserializers = deserializers;
+ this.applicationEventHandler = applicationEventHandler;
+ this.assignors = assignors;
+ this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer");
}
/**
- * poll implementation using {@link EventHandler}.
+ * poll implementation using {@link ApplicationEventHandler}.
* 1. Poll for background events. If there's a fetch response event, process the record and return it. If it is
* another type of event, process it.
* 2. Send fetches if needed.
@@ -193,76 +327,33 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public ConsumerRecords<K, V> poll(final Duration timeout) {
Timer timer = time.timer(timeout);
+
try {
- do {
- if (!eventHandler.isEmpty()) {
- final Optional<BackgroundEvent> backgroundEvent = eventHandler.poll();
- // processEvent() may process 3 types of event:
- // 1. Errors
- // 2. Callback Invocation
- // 3. Fetch responses
- // Errors will be handled or rethrown.
- // Callback invocation will trigger callback function execution, which is blocking until completion.
- // Successful fetch responses will be added to the completedFetches in the fetcher, which will then
- // be processed in the collectFetches().
- backgroundEvent.ifPresent(event -> processEvent(event, timeout));
- }
+ kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs());
+
+ if (subscriptions.hasNoSubscriptionOrUserAssignment()) {
+ throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions");
+ }
- updateFetchPositionsIfNeeded(timer);
+ do {
+ updateAssignmentMetadataIfNeeded(timer);
+ final Fetch<K, V> fetch = pollForFetches(timer);
- // The idea here is to have the background thread sending fetches autonomously, and the fetcher
- // uses the poll loop to retrieve successful fetchResponse and process them on the polling thread.
- final Fetch<K, V> fetch = collectFetches();
if (!fetch.isEmpty()) {
- return processFetchResults(fetch);
+ if (fetch.records().isEmpty()) {
+ log.trace("Returning empty records from `poll()` "
+ + "since the consumer's position has advanced for at least one topic partition");
+ }
+
+ return interceptors.onConsume(new ConsumerRecords<>(fetch.records()));
}
// We will wait for retryBackoffMs
- } while (time.timer(timeout).notExpired());
- } catch (final Exception e) {
- throw new RuntimeException(e);
- }
- // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask();
-
- return ConsumerRecords.empty();
- }
-
- /**
- * Set the fetch position to the committed position (if there is one) or reset it using the
- * offset reset policy the user has configured (if partitions require reset)
- *
- * @return true if the operation completed without timing out
- * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
- * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is
- * defined
- */
- private boolean updateFetchPositionsIfNeeded(final Timer timer) {
- // Validate positions using the partition leader end offsets, to detect if any partition
- // has been truncated due to a leader change. This will trigger an OffsetForLeaderEpoch
- // request, retrieve the partition end offsets, and validate the current position against it.
- ValidatePositionsApplicationEvent validatePositionsEvent = new ValidatePositionsApplicationEvent();
- eventHandler.add(validatePositionsEvent);
-
- // Reset positions using committed offsets retrieved from the group coordinator, for any
- // partitions which do not have a valid position and are not awaiting reset. This will
- // trigger an OffsetFetch request and update positions with the offsets retrieved. This
- // will only do a coordinator lookup if there are partitions which have missing
- // positions, so a consumer with manually assigned partitions can avoid a coordinator
- // dependence by always ensuring that assigned partitions have an initial position.
- if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer))
- return false;
-
- // If there are partitions still needing a position and a reset policy is defined,
- // request reset using the default policy. If no reset strategy is defined and there
- // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception.
- subscriptions.resetInitializingPositions();
+ } while (timer.notExpired());
- // Reset positions using partition offsets retrieved from the leader, for any partitions
- // which are awaiting reset. This will trigger a ListOffset request, retrieve the
- // partition offsets according to the strategy (ex. earliest, latest), and update the
- // positions.
- ResetPositionsApplicationEvent resetPositionsEvent = new ResetPositionsApplicationEvent();
- eventHandler.add(resetPositionsEvent);
- return true;
+ return ConsumerRecords.empty();
+ } finally {
+ kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs());
+ }
}
/**
@@ -274,20 +365,6 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
commitSync(Duration.ofMillis(defaultApiTimeoutMs));
}
- private void processEvent(final BackgroundEvent backgroundEvent, final Duration timeout) {
- // stubbed class
- }
-
- private ConsumerRecords<K, V> processFetchResults(final Fetch<K, V> fetch) {
- // stubbed class
- return ConsumerRecords.empty();
- }
-
- private Fetch<K, V> collectFetches() {
- // stubbed class
- return Fetch.empty();
- }
-
/**
* This method sends a commit event to the EventHandler and return.
*/
@@ -324,50 +401,96 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
// the task can only be woken up if the top level API call is commitSync
wakeupTrigger.setActiveTask(commitEvent.future());
}
- eventHandler.add(commitEvent);
+ applicationEventHandler.add(commitEvent);
return commitEvent.future();
}
@Override
public void seek(TopicPartition partition, long offset) {
- throw new KafkaException("method not implemented");
+ if (offset < 0)
+ throw new IllegalArgumentException("seek offset must not be a negative number");
+
+ log.info("Seeking to offset {} for partition {}", offset, partition);
+ SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition(
+ offset,
+ Optional.empty(), // This will ensure we skip validation
+ metadata.currentLeader(partition));
+ subscriptions.seekUnvalidated(partition, newPosition);
}
@Override
public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) {
- throw new KafkaException("method not implemented");
+ long offset = offsetAndMetadata.offset();
+ if (offset < 0) {
+ throw new IllegalArgumentException("seek offset must not be a negative number");
+ }
+
+ if (offsetAndMetadata.leaderEpoch().isPresent()) {
+ log.info("Seeking to offset {} for partition {} with epoch {}",
+ offset, partition, offsetAndMetadata.leaderEpoch().get());
+ } else {
+ log.info("Seeking to offset {} for partition {}", offset, partition);
+ }
+ Metadata.LeaderAndEpoch currentLeaderAndEpoch = metadata.currentLeader(partition);
+ SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition(
+ offsetAndMetadata.offset(),
+ offsetAndMetadata.leaderEpoch(),
+ currentLeaderAndEpoch);
+ updateLastSeenEpochIfNewer(partition, offsetAndMetadata);
+ subscriptions.seekUnvalidated(partition, newPosition);
}
@Override
public void seekToBeginning(Collection<TopicPartition> partitions) {
- throw new KafkaException("method not implemented");
+ if (partitions == null)
+ throw new IllegalArgumentException("Partitions collection cannot be null");
+
+ Collection<TopicPartition> parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions;
+ subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST);
}
@Override
public void seekToEnd(Collection<TopicPartition> partitions) {
- throw new KafkaException("method not implemented");
+ if (partitions == null)
+ throw new IllegalArgumentException("Partitions collection cannot be null");
+
+ Collection<TopicPartition> parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions;
+ subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST);
}
@Override
public long position(TopicPartition partition) {
- throw new KafkaException("method not implemented");
+ return position(partition, Duration.ofMillis(defaultApiTimeoutMs));
}
@Override
public long position(TopicPartition partition, Duration timeout) {
- throw new KafkaException("method not implemented");
+ if (!subscriptions.isAssigned(partition))
+ throw new IllegalStateException("You can only check the position for partitions assigned to this consumer.");
+
+ Timer timer = time.timer(timeout);
+ do {
+ SubscriptionState.FetchPosition position = subscriptions.validPosition(partition);
+ if (position != null)
+ return position.offset;
+
+ updateFetchPositions(timer);
+ } while (timer.notExpired());
+
+ throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " +
+ "for partition " + partition + " could be determined");
}
@Override
@Deprecated
public OffsetAndMetadata committed(TopicPartition partition) {
- throw new KafkaException("method not implemented");
+ return committed(partition, Duration.ofMillis(defaultApiTimeoutMs));
}
@Override
@Deprecated
public OffsetAndMetadata committed(TopicPartition partition, Duration timeout) {
- throw new KafkaException("method not implemented");
+ return committed(Collections.singleton(partition), timeout).get(partition);
}
@Override
@@ -386,7 +509,7 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(partitions);
wakeupTrigger.setActiveTask(event.future());
try {
- return eventHandler.addAndGet(event, time.timer(timeout));
+ return applicationEventHandler.addAndGet(event, time.timer(timeout));
} finally {
wakeupTrigger.clearActiveTask();
}
@@ -401,12 +524,12 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public Map<MetricName, ? extends Metric> metrics() {
- throw new KafkaException("method not implemented");
+ return Collections.unmodifiableMap(metrics.metrics());
}
@Override
public List<PartitionInfo> partitionsFor(String topic) {
- throw new KafkaException("method not implemented");
+ return partitionsFor(topic, Duration.ofMillis(defaultApiTimeoutMs));
}
@Override
@@ -416,7 +539,7 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public Map<String, List<PartitionInfo>> listTopics() {
- throw new KafkaException("method not implemented");
+ return listTopics(Duration.ofMillis(defaultApiTimeoutMs));
}
@Override
@@ -426,17 +549,23 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public Set<TopicPartition> paused() {
- throw new KafkaException("method not implemented");
+ return Collections.unmodifiableSet(subscriptions.pausedPartitions());
}
@Override
public void pause(Collection<TopicPartition> partitions) {
- throw new KafkaException("method not implemented");
+ log.debug("Pausing partitions {}", partitions);
+ for (TopicPartition partition: partitions) {
+ subscriptions.pause(partition);
+ }
}
@Override
public void resume(Collection<TopicPartition> partitions) {
- throw new KafkaException("method not implemented");
+ log.debug("Resuming partitions {}", partitions);
+ for (TopicPartition partition: partitions) {
+ subscriptions.resume(partition);
+ }
}
@Override
@@ -469,7 +598,7 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
if (timeout.toMillis() == 0L)
return listOffsetsEvent.emptyResult();
- return eventHandler.addAndGet(listOffsetsEvent, time.timer(timeout));
+ return applicationEventHandler.addAndGet(listOffsetsEvent, time.timer(timeout));
}
@Override
@@ -502,20 +631,42 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
if (partitions.isEmpty()) {
return Collections.emptyMap();
}
- Map<TopicPartition, Long> timestampToSearch =
- partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> timestamp));
- final ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent(
+ Map<TopicPartition, Long> timestampToSearch = partitions
+ .stream()
+ .collect(Collectors.toMap(Function.identity(), tp -> timestamp));
+ ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent(
timestampToSearch,
false);
- Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestampMap =
- eventHandler.addAndGet(listOffsetsEvent, time.timer(timeout));
- return offsetAndTimestampMap.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(),
- e -> e.getValue().offset()));
+ Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestampMap = applicationEventHandler.addAndGet(
+ listOffsetsEvent,
+ time.timer(timeout));
+ return offsetAndTimestampMap
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset()));
}
@Override
public OptionalLong currentLag(TopicPartition topicPartition) {
- throw new KafkaException("method not implemented");
+ final Long lag = subscriptions.partitionLag(topicPartition, isolationLevel);
+
+ // if the log end offset is not known and hence cannot return lag and there is
+ // no in-flight list offset requested yet,
+ // issue a list offset request for that partition so that next time
+ // we may get the answer; we do not need to wait for the return value
+ // since we would not try to poll the network client synchronously
+ if (lag == null) {
+ if (subscriptions.partitionEndOffset(topicPartition, isolationLevel) == null &&
+ !subscriptions.partitionEndOffsetRequested(topicPartition)) {
+ log.info("Requesting the log end offset for {} in order to compute lag", topicPartition);
+ subscriptions.requestPartitionEndOffset(topicPartition);
+ endOffsets(Collections.singleton(topicPartition), Duration.ofMillis(0));
+ }
+
+ return OptionalLong.empty();
+ }
+
+ return OptionalLong.of(lag);
}
@Override
@@ -540,11 +691,37 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public void close(Duration timeout) {
+ if (timeout.toMillis() < 0)
+ throw new IllegalArgumentException("The timeout cannot be negative.");
+
+ try {
+ if (!closed) {
+ // need to close before setting the flag since the close function
+ // itself may trigger rebalance callback that needs the consumer to be open still
+ close(timeout, false);
+ }
+ } finally {
+ closed = true;
+ }
+ }
+
+ private void close(Duration timeout, boolean swallowException) {
+ log.trace("Closing the Kafka consumer");
AtomicReference<Throwable> firstException = new AtomicReference<>();
- closeQuietly(this.eventHandler, "event handler", firstException);
+
+ if (applicationEventHandler != null)
+ closeQuietly(() -> applicationEventHandler.close(timeout), "Failed to close application event handler with a timeout(ms)=" + timeout, firstException);
+
+ closeQuietly(fetchBuffer, "Failed to close the fetch buffer", firstException);
+ closeQuietly(interceptors, "consumer interceptors", firstException);
+ closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException);
+ closeQuietly(metrics, "consumer metrics", firstException);
+ closeQuietly(deserializers, "consumer deserializers", firstException);
+
+ AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics);
log.debug("Kafka consumer has been closed");
Throwable exception = firstException.get();
- if (exception != null) {
+ if (exception != null && !swallowException) {
if (exception instanceof InterruptException) {
throw (InterruptException) exception;
}
@@ -575,19 +752,14 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets, Duration timeout) {
- CompletableFuture<Void> commitFuture = commit(offsets, true);
+ long commitStart = time.nanoseconds();
try {
- commitFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
- } catch (final TimeoutException e) {
- throw new org.apache.kafka.common.errors.TimeoutException(e);
- } catch (final InterruptedException e) {
- throw new InterruptException(e);
- } catch (final ExecutionException e) {
- if (e.getCause() instanceof WakeupException)
- throw new WakeupException();
- throw new KafkaException(e);
+ CompletableFuture<Void> commitFuture = commit(offsets, true);
+ offsets.forEach(this::updateLastSeenEpochIfNewer);
+ ConsumerUtils.getResult(commitFuture, time.timer(timeout));
} finally {
wakeupTrigger.clearActiveTask();
+ kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart);
}
}
@@ -598,7 +770,7 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
@Override
public Set<TopicPartition> assignment() {
- return Collections.unmodifiableSet(this.subscriptions.assignedPartitions());
+ return Collections.unmodifiableSet(subscriptions.assignedPartitions());
}
/**
@@ -608,17 +780,43 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
*/
@Override
public Set<String> subscription() {
- return Collections.unmodifiableSet(this.subscriptions.subscription());
+ return Collections.unmodifiableSet(subscriptions.subscription());
}
@Override
public void subscribe(Collection<String> topics) {
- throw new KafkaException("method not implemented");
+ subscribe(topics, new NoOpConsumerRebalanceListener());
}
@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
- throw new KafkaException("method not implemented");
+ maybeThrowInvalidGroupIdException();
+ if (topics == null)
+ throw new IllegalArgumentException("Topic collection to subscribe to cannot be null");
+ if (topics.isEmpty()) {
+ // treat subscribing to empty topic list as the same as unsubscribing
+ unsubscribe();
+ } else {
+ for (String topic : topics) {
+ if (isBlank(topic))
+ throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic");
+ }
+
+ throwIfNoAssignorsConfigured();
+
+ // Clear the buffered data which are not a part of newly assigned topics
+ final Set<TopicPartition> currentTopicPartitions = new HashSet<>();
+
+ for (TopicPartition tp : subscriptions.assignedPartitions()) {
+ if (topics.contains(tp.topic()))
+ currentTopicPartitions.add(tp);
+ }
+
+ fetchBuffer.retainAll(currentTopicPartitions);
+ log.info("Subscribed to topic(s): {}", join(topics, ", "));
+ if (subscriptions.subscribe(new HashSet<>(topics), callback))
+ metadata.requestUpdateForNewTopics();
+ }
}
@Override
@@ -628,8 +826,7 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
}
if (partitions.isEmpty()) {
- // TODO: implementation of unsubscribe() will be included in forthcoming commits.
- // this.unsubscribe();
+ unsubscribe();
return;
}
@@ -639,38 +836,75 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic");
}
- // TODO: implementation of refactored Fetcher will be included in forthcoming commits.
- // fetcher.clearBufferedDataForUnassignedPartitions(partitions);
+ // Clear the buffered data which are not a part of newly assigned topics
+ final Set<TopicPartition> currentTopicPartitions = new HashSet<>();
+
+ for (TopicPartition tp : subscriptions.assignedPartitions()) {
+ if (partitions.contains(tp))
+ currentTopicPartitions.add(tp);
+ }
+
+ fetchBuffer.retainAll(currentTopicPartitions);
// assignment change event will trigger autocommit if it is configured and the group id is specified. This is
// to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will
- // be no following rebalance
- eventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds()));
+ // be no following rebalance.
+ //
+ // See the ApplicationEventProcessor.process() method that handles this event for more detail.
+ applicationEventHandler.add(new AssignmentChangeApplicationEvent(subscriptions.allConsumed(), time.milliseconds()));
log.info("Assigned to partition(s): {}", join(partitions, ", "));
- if (this.subscriptions.assignFromUser(new HashSet<>(partitions)))
- eventHandler.add(new NewTopicsMetadataUpdateRequestEvent());
+ if (subscriptions.assignFromUser(new HashSet<>(partitions)))
+ applicationEventHandler.add(new NewTopicsMetadataUpdateRequestEvent());
}
@Override
- public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) {
- throw new KafkaException("method not implemented");
+ public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) {
+ maybeThrowInvalidGroupIdException();
+ if (pattern == null || pattern.toString().isEmpty())
+ throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ?
+ "null" : "empty"));
+
+ throwIfNoAssignorsConfigured();
+ log.info("Subscribed to pattern: '{}'", pattern);
+ subscriptions.subscribe(pattern, listener);
+ updatePatternSubscription(metadata.fetch());
+ metadata.requestUpdateForNewTopics();
+ }
+
+ /**
+ * TODO: remove this when we implement the KIP-848 protocol.
+ *
+ * <p>
+ * The contents of this method are shamelessly stolen from
+ * {@link ConsumerCoordinator#updatePatternSubscription(Cluster)} and are used here because we won't have access
+ * to a {@link ConsumerCoordinator} in this code. Perhaps it could be moved to a ConsumerUtils class?
+ *
+ * @param cluster Cluster from which we get the topics
+ */
+ private void updatePatternSubscription(Cluster cluster) {
+ final Set<String> topicsToSubscribe = cluster.topics().stream()
+ .filter(subscriptions::matchesSubscribedPattern)
+ .collect(Collectors.toSet());
+ if (subscriptions.subscribeFromPattern(topicsToSubscribe))
+ metadata.requestUpdateForNewTopics();
}
@Override
public void subscribe(Pattern pattern) {
- throw new KafkaException("method not implemented");
+ subscribe(pattern, new NoOpConsumerRebalanceListener());
}
@Override
public void unsubscribe() {
- throw new KafkaException("method not implemented");
+ fetchBuffer.retainAll(Collections.emptySet());
+ subscriptions.unsubscribe();
}
@Override
@Deprecated
- public ConsumerRecords<K, V> poll(long timeout) {
- throw new KafkaException("method not implemented");
+ public ConsumerRecords<K, V> poll(final long timeoutMs) {
+ return poll(Duration.ofMillis(timeoutMs));
}
// Visible for testing
@@ -678,17 +912,99 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
return wakeupTrigger;
}
- private static <K, V> ClusterResourceListeners configureClusterResourceListeners(
- final Deserializer<K> keyDeserializer,
- final Deserializer<V> valueDeserializer,
- final List<?>... candidateLists) {
- ClusterResourceListeners clusterResourceListeners = new ClusterResourceListeners();
- for (List<?> candidateList: candidateLists)
- clusterResourceListeners.maybeAddAll(candidateList);
+ private Fetch<K, V> pollForFetches(Timer timer) {
+ long pollTimeout = timer.remainingMs();
+
+ // if data is available already, return it immediately
+ final Fetch<K, V> fetch = collectFetch();
+ if (!fetch.isEmpty()) {
+ return fetch;
+ }
+
+ // We do not want to be stuck blocking in poll if we are missing some positions
+ // since the offset lookup may be backing off after a failure
+
+ // NOTE: the use of cachedSubscriptionHasAllFetchPositions means we MUST call
+ // updateAssignmentMetadataIfNeeded before this method.
+ if (!cachedSubscriptionHasAllFetchPositions && pollTimeout > retryBackoffMs) {
+ pollTimeout = retryBackoffMs;
+ }
+
+ log.trace("Polling for fetches with timeout {}", pollTimeout);
+
+ Timer pollTimer = time.timer(pollTimeout);
+
+ // Wait a bit for some fetched data to arrive, as there may not be anything immediately available. Note the
+ // use of a shorter, dedicated "pollTimer" here which updates "timer" so that calling method (poll) will
+ // correctly handle the overall timeout.
+ try {
+ fetchBuffer.awaitNotEmpty(pollTimer);
+ } catch (InterruptException e) {
+ log.trace("Timeout during fetch", e);
+ } finally {
+ timer.update(pollTimer.currentTimeMs());
+ }
+
+ return collectFetch();
+ }
+
+ /**
+ * Perform the "{@link FetchCollector#collectFetch(FetchBuffer) fetch collection}" step by reading raw data out
+ * of the {@link #fetchBuffer}, converting it to a well-formed {@link CompletedFetch}, validating that it and
+ * the internal {@link SubscriptionState state} are correct, and then converting it all into a {@link Fetch}
+ * for returning.
+ *
+ * <p/>
+ *
+ * This method will {@link ApplicationEventHandler#wakeupNetworkThread() wake up} the {@link ConsumerNetworkThread} before
+ * retuning. This is done as an optimization so that the <em>next round of data can be pre-fetched</em>.
+ */
+ private Fetch<K, V> collectFetch() {
+ final Fetch<K, V> fetch = fetchCollector.collectFetch(fetchBuffer);
+
+ // Notify the network thread to wake up and start the next round of fetching.
+ applicationEventHandler.wakeupNetworkThread();
+
+ return fetch;
+ }
+ /**
+ * Set the fetch position to the committed position (if there is one)
+ * or reset it using the offset reset policy the user has configured.
+ *
+ * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details
+ * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is
+ * defined
+ * @return true iff the operation completed without timing out
+ */
+ private boolean updateFetchPositions(final Timer timer) {
+ // Validate positions using the partition leader end offsets, to detect if any partition
+ // has been truncated due to a leader change. This will trigger an OffsetForLeaderEpoch
+ // request, retrieve the partition end offsets, and validate the current position against it.
+ applicationEventHandler.add(new ValidatePositionsApplicationEvent());
+
+ cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions();
+ if (cachedSubscriptionHasAllFetchPositions) return true;
+
+ // Reset positions using committed offsets retrieved from the group coordinator, for any
+ // partitions which do not have a valid position and are not awaiting reset. This will
+ // trigger an OffsetFetch request and update positions with the offsets retrieved. This
+ // will only do a coordinator lookup if there are partitions which have missing
+ // positions, so a consumer with manually assigned partitions can avoid a coordinator
+ // dependence by always ensuring that assigned partitions have an initial position.
+ if (isCommittedOffsetsManagementEnabled() && !initWithCommittedOffsetsIfNeeded(timer))
+ return false;
+
+ // If there are partitions still needing a position and a reset policy is defined,
+ // request reset using the default policy. If no reset strategy is defined and there
+ // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception.
+ subscriptions.resetInitializingPositions();
- clusterResourceListeners.maybeAdd(keyDeserializer);
- clusterResourceListeners.maybeAdd(valueDeserializer);
- return clusterResourceListeners;
+ // Reset positions using partition offsets retrieved from the leader, for any partitions
+ // which are awaiting reset. This will trigger a ListOffset request, retrieve the
+ // partition offsets according to the strategy (ex. earliest, latest), and update the
+ // positions.
+ applicationEventHandler.add(new ResetPositionsApplicationEvent());
+ return true;
}
/**
@@ -706,20 +1022,21 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
* @param timer Timer bounding how long this method can block
* @return true iff the operation completed within the timeout
*/
- private boolean refreshCommittedOffsetsIfNeeded(Timer timer) {
+ private boolean initWithCommittedOffsetsIfNeeded(Timer timer) {
final Set<TopicPartition> initializingPartitions = subscriptions.initializingPartitions();
log.debug("Refreshing committed offsets for partitions {}", initializingPartitions);
try {
- final Map<TopicPartition, OffsetAndMetadata> offsets = eventHandler.addAndGet(new OffsetFetchApplicationEvent(initializingPartitions), timer);
- return ConsumerUtils.refreshCommittedOffsets(offsets, this.metadata, this.subscriptions);
- } catch (org.apache.kafka.common.errors.TimeoutException e) {
+ final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(initializingPartitions);
+ final Map<TopicPartition, OffsetAndMetadata> offsets = applicationEventHandler.addAndGet(event, timer);
+ refreshCommittedOffsets(offsets, metadata, subscriptions);
+ return true;
+ } catch (TimeoutException e) {
log.error("Couldn't refresh committed offsets before timeout expired");
return false;
}
}
-
// This is here temporary as we don't have public access to the ConsumerConfig in this module.
public static Map<String, Object> appendDeserializerToConfig(Map<String, Object> configs,
Deserializer<?> keyDeserializer,
@@ -737,6 +1054,17 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
return newConfigs;
}
+ private void throwIfNoAssignorsConfigured() {
+ if (assignors.isEmpty())
+ throw new IllegalStateException("Must configure at least one partition assigner class name to " +
+ ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG + " configuration property");
+ }
+
+ private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) {
+ if (offsetAndMetadata != null)
+ offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch));
+ }
+
private class DefaultOffsetCommitCallback implements OffsetCommitCallback {
@Override
public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
@@ -744,4 +1072,13 @@ public class PrototypeAsyncConsumer<K, V> implements Consumer<K, V> {
log.error("Offset commit with offsets {} failed", offsets, exception);
}
}
+
+ boolean updateAssignmentMetadataIfNeeded(Timer timer) {
+ backgroundEventProcessor.process();
+
+ // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as
+ // in the previous implementation, because it will eventually involve group coordination
+ // logic
+ return updateFetchPositions(timer);
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java
index 4655f5ef704..8592035dda2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java
@@ -16,13 +16,50 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult;
+import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY;
+
/**
* {@code PollResult} consist of {@code UnsentRequest} if there are requests to send; otherwise, return the time till
* the next poll event.
*/
public interface RequestManager {
+
+ /**
+ * During normal operation of the {@link Consumer}, a request manager may need to send out network requests.
+ * Implementations can return {@link PollResult their need for network I/O} by returning the requests here.
+ * This method is called within a single-threaded context from
+ * {@link ConsumerNetworkThread the consumer's network I/O thread}. As such, there should be no need for
+ * synchronization protection in this method's implementation.
+ *
+ * <p/>
+ *
+ * <em>Note</em>: no network I/O occurs in this method. The method itself should not block for any reason. This
+ * method is called from the consumer's network I/O thread, so quick execution of this method in <em>all</em>
+ * request managers is critical to ensure that we can heartbeat in a timely fashion.
+ *
+ * @param currentTimeMs The current system time at which the method was called; useful for determining if
+ * time-sensitive operations should be performed
+ */
PollResult poll(long currentTimeMs);
+ /**
+ * On shutdown of the {@link Consumer}, a request manager may need to send out network requests. Implementations
+ * can signal that by returning the {@link PollResult close} requests here. Like {@link #poll(long)}, this method
+ * is called within a single-threaded context from {@link ConsumerNetworkThread the consumer's network I/O thread}.
+ * As such, there should be no need for synchronization protection in this method's implementation.
+ *
+ * <p/>
+ *
+ * <em>Note</em>: no network I/O occurs in this method. The method itself should not block for any reason. This
+ * method is called as an (indirect) result of {@link Consumer#close() the consumer's close method} being invoked.
+ * (Note that it is still invoked on the consumer's network I/O thread). Quick execution of this method in
+ * <em>all</em> request managers is critical to ensure that we can complete as many of the consumer's shutdown
+ * tasks as possible within the user-provided timeout.
+ */
+ default PollResult pollOnClose() {
+ return EMPTY;
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java
index 5a397532fe3..b600a528164 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java
@@ -16,10 +16,25 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.ApiVersions;
+import org.apache.kafka.clients.GroupRebalanceConfig;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.common.internals.IdempotentCloser;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Time;
+import org.slf4j.Logger;
+
+import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.function.Supplier;
+
+import static org.apache.kafka.common.utils.Utils.closeQuietly;
import static java.util.Objects.requireNonNull;
@@ -28,25 +43,31 @@ import static java.util.Objects.requireNonNull;
* This allows callers to both use the specific {@link RequestManager} instance, or to iterate over the list via
* the {@link #entries()} method.
*/
-public class RequestManagers {
+public class RequestManagers implements Closeable {
+ private final Logger log;
public final Optional<CoordinatorRequestManager> coordinatorRequestManager;
public final Optional<CommitRequestManager> commitRequestManager;
- private final Optional<HeartbeatRequestManager> heartbeatRequestManager;
+ public final Optional<HeartbeatRequestManager> heartbeatRequestManager;
public final OffsetsRequestManager offsetsRequestManager;
public final TopicMetadataRequestManager topicMetadataRequestManager;
+ public final FetchRequestManager fetchRequestManager;
private final List<Optional<? extends RequestManager>> entries;
+ private final IdempotentCloser closer = new IdempotentCloser();
- public RequestManagers(OffsetsRequestManager offsetsRequestManager,
+ public RequestManagers(LogContext logContext,
+ OffsetsRequestManager offsetsRequestManager,
TopicMetadataRequestManager topicMetadataRequestManager,
+ FetchRequestManager fetchRequestManager,
Optional<CoordinatorRequestManager> coordinatorRequestManager,
Optional<CommitRequestManager> commitRequestManager,
Optional<HeartbeatRequestManager> heartbeatRequestManager) {
- this.offsetsRequestManager = requireNonNull(offsetsRequestManager,
- "OffsetsRequestManager cannot be null");
+ this.log = logContext.logger(RequestManagers.class);
+ this.offsetsRequestManager = requireNonNull(offsetsRequestManager, "OffsetsRequestManager cannot be null");
this.coordinatorRequestManager = coordinatorRequestManager;
this.commitRequestManager = commitRequestManager;
this.topicMetadataRequestManager = topicMetadataRequestManager;
+ this.fetchRequestManager = fetchRequestManager;
this.heartbeatRequestManager = heartbeatRequestManager;
List<Optional<? extends RequestManager>> list = new ArrayList<>();
@@ -55,10 +76,111 @@ public class RequestManagers {
list.add(heartbeatRequestManager);
list.add(Optional.of(offsetsRequestManager));
list.add(Optional.of(topicMetadataRequestManager));
+ list.add(Optional.of(fetchRequestManager));
entries = Collections.unmodifiableList(list);
}
public List<Optional<? extends RequestManager>> entries() {
return entries;
}
+
+ @Override
+ public void close() {
+ closer.close(
+ () -> {
+ log.debug("Closing RequestManagers");
+
+ entries.stream()
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .filter(rm -> rm instanceof Closeable)
+ .map(rm -> (Closeable) rm)
+ .forEach(c -> closeQuietly(c, c.getClass().getSimpleName()));
+ log.debug("RequestManagers has been closed");
+ },
+ () -> log.debug("RequestManagers was already closed")
+ );
+ }
+
+ /**
+ * Creates a {@link Supplier} for deferred creation during invocation by
+ * {@link ConsumerNetworkThread}.
+ */
+ public static Supplier<RequestManagers> supplier(final Time time,
+ final LogContext logContext,
+ final BlockingQueue<BackgroundEvent> backgroundEventQueue,
+ final ConsumerMetadata metadata,
+ final SubscriptionState subscriptions,
+ final FetchBuffer fetchBuffer,
+ final ConsumerConfig config,
+ final GroupRebalanceConfig groupRebalanceConfig,
+ final ApiVersions apiVersions,
+ final FetchMetricsManager fetchMetricsManager,
+ final Supplier<NetworkClientDelegate> networkClientDelegateSupplier) {
+ return new CachedSupplier<RequestManagers>() {
+ @Override
+ protected RequestManagers create() {
+ final NetworkClientDelegate networkClientDelegate = networkClientDelegateSupplier.get();
+ final BackgroundEventHandler backgroundEventHandler = new BackgroundEventHandler(logContext, backgroundEventQueue);
+ final FetchConfig fetchConfig = new FetchConfig(config);
+ long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
+ long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG);
+ final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG);
+ final OffsetsRequestManager listOffsets = new OffsetsRequestManager(subscriptions,
+ metadata,
+ fetchConfig.isolationLevel,
+ time,
+ retryBackoffMs,
+ requestTimeoutMs,
+ apiVersions,
+ networkClientDelegate,
+ backgroundEventHandler,
+ logContext);
+ final FetchRequestManager fetch = new FetchRequestManager(logContext,
+ time,
+ metadata,
+ subscriptions,
+ fetchConfig,
+ fetchBuffer,
+ fetchMetricsManager,
+ networkClientDelegate);
+ final TopicMetadataRequestManager topic = new TopicMetadataRequestManager(
+ logContext,
+ config);
+ HeartbeatRequestManager heartbeatRequestManager = null;
+ CoordinatorRequestManager coordinator = null;
+ CommitRequestManager commit = null;
+
+ if (groupRebalanceConfig != null && groupRebalanceConfig.groupId != null) {
+ final GroupState groupState = new GroupState(groupRebalanceConfig);
+ coordinator = new CoordinatorRequestManager(time,
+ logContext,
+ retryBackoffMs,
+ retryBackoffMaxMs,
+ backgroundEventHandler,
+ groupState.groupId);
+ commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState);
+ MembershipManager membershipManager = new MembershipManagerImpl(groupState.groupId, logContext);
+ heartbeatRequestManager = new HeartbeatRequestManager(
+ logContext,
+ time,
+ config,
+ coordinator,
+ subscriptions,
+ membershipManager,
+ backgroundEventHandler);
+ }
+
+ return new RequestManagers(
+ logContext,
+ listOffsets,
+ topic,
+ fetch,
+ Optional.ofNullable(coordinator),
+ Optional.ofNullable(commit),
+ Optional.ofNullable(heartbeatRequestManager)
+ );
+ }
+ };
+ }
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java
index 2443e4e30e7..f62794280f4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java
@@ -40,6 +40,8 @@ import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
+import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY;
+
/**
* <p>
* Manages the state of topic metadata requests. This manager returns a
@@ -84,9 +86,7 @@ public class TopicMetadataRequestManager implements RequestManager {
.filter(Optional::isPresent)
.map(Optional::get)
.collect(Collectors.toList());
- return requests.isEmpty() ?
- new NetworkClientDelegate.PollResult(Long.MAX_VALUE, new ArrayList<>()) :
- new NetworkClientDelegate.PollResult(0, Collections.unmodifiableList(requests));
+ return requests.isEmpty() ? EMPTY : new NetworkClientDelegate.PollResult(0, requests);
}
/**
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java
index bf9bb1d4962..133836da3b7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java
@@ -24,7 +24,7 @@ import java.util.Objects;
public abstract class ApplicationEvent {
public enum Type {
- NOOP, COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE,
+ COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE,
LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java
new file mode 100644
index 00000000000..2917d507d7f
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals.events;
+
+import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread;
+import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate;
+import org.apache.kafka.clients.consumer.internals.RequestManagers;
+import org.apache.kafka.common.internals.IdempotentCloser;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Timer;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Objects;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+/**
+ * An event handler that receives {@link ApplicationEvent application events} from the application thread which
+ * are then readable from the {@link ApplicationEventProcessor} in the {@link ConsumerNetworkThread network thread}.
+ */
+public class ApplicationEventHandler implements Closeable {
+
+ private final Logger log;
+ private final BlockingQueue<ApplicationEvent> applicationEventQueue;
+ private final ConsumerNetworkThread networkThread;
+ private final IdempotentCloser closer = new IdempotentCloser();
+
+ public ApplicationEventHandler(final LogContext logContext,
+ final Time time,
+ final BlockingQueue<ApplicationEvent> applicationEventQueue,
+ final Supplier<ApplicationEventProcessor> applicationEventProcessorSupplier,
+ final Supplier<NetworkClientDelegate> networkClientDelegateSupplier,
+ final Supplier<RequestManagers> requestManagersSupplier) {
+ this.log = logContext.logger(ApplicationEventHandler.class);
+ this.applicationEventQueue = applicationEventQueue;
+ this.networkThread = new ConsumerNetworkThread(logContext,
+ time,
+ applicationEventProcessorSupplier,
+ networkClientDelegateSupplier,
+ requestManagersSupplier);
+ this.networkThread.start();
+ }
+
+ /**
+ * Add an {@link ApplicationEvent} to the handler and then internally invoke {@link #wakeupNetworkThread}
+ * to alert the network I/O thread that it has something to process.
+ *
+ * @param event An {@link ApplicationEvent} created by the application thread
+ */
+ public void add(final ApplicationEvent event) {
+ Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null");
+ log.trace("Enqueued event: {}", event);
+ applicationEventQueue.add(event);
+ wakeupNetworkThread();
+ }
+
+ /**
+ * Wakeup the {@link ConsumerNetworkThread network I/O thread} to pull the next event(s) from the queue.
+ */
+ public void wakeupNetworkThread() {
+ networkThread.wakeup();
+ }
+
+ /**
+ * Add a {@link CompletableApplicationEvent} to the handler. The method blocks waiting for the result, and will
+ * return the result value upon successful completion; otherwise throws an error.
+ *
+ * <p/>
+ *
+ * See {@link CompletableApplicationEvent#get(Timer)} and {@link Future#get(long, TimeUnit)} for more details.
+ *
+ * @param event A {@link CompletableApplicationEvent} created by the polling thread
+ * @param timer Timer for which to wait for the event to complete
+ * @return Value that is the result of the event
+ * @param <T> Type of return value of the event
+ */
+ public <T> T addAndGet(final CompletableApplicationEvent<T> event, final Timer timer) {
+ Objects.requireNonNull(event, "CompletableApplicationEvent provided to addAndGet must be non-null");
+ Objects.requireNonNull(timer, "Timer provided to addAndGet must be non-null");
+ add(event);
+ return event.get(timer);
+ }
+
+ @Override
+ public void close() {
+ close(Duration.ZERO);
+ }
+
+ public void close(final Duration timeout) {
+ closer.close(
+ () -> Utils.closeQuietly(() -> networkThread.close(timeout), "consumer network thread"),
+ () -> log.warn("The application event handler was already closed")
+ );
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java
index f2f55860004..ccbdd21b9dc 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java
@@ -17,146 +17,188 @@
package org.apache.kafka.clients.consumer.internals.events;
import org.apache.kafka.clients.consumer.OffsetAndTimestamp;
+import org.apache.kafka.clients.consumer.internals.CachedSupplier;
import org.apache.kafka.clients.consumer.internals.CommitRequestManager;
import org.apache.kafka.clients.consumer.internals.ConsumerMetadata;
+import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread;
import org.apache.kafka.clients.consumer.internals.RequestManagers;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.utils.LogContext;
+import org.slf4j.Logger;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
-public class ApplicationEventProcessor {
-
- private final BlockingQueue<BackgroundEvent> backgroundEventQueue;
+/**
+ * An {@link EventProcessor} that is created and executes in the {@link ConsumerNetworkThread network thread}
+ * which processes {@link ApplicationEvent application events} generated by the application thread.
+ */
+public class ApplicationEventProcessor extends EventProcessor<ApplicationEvent> {
+ private final Logger log;
private final ConsumerMetadata metadata;
-
private final RequestManagers requestManagers;
- public ApplicationEventProcessor(final BlockingQueue<BackgroundEvent> backgroundEventQueue,
+ public ApplicationEventProcessor(final LogContext logContext,
+ final BlockingQueue<ApplicationEvent> applicationEventQueue,
final RequestManagers requestManagers,
final ConsumerMetadata metadata) {
- this.backgroundEventQueue = backgroundEventQueue;
+ super(logContext, applicationEventQueue);
+ this.log = logContext.logger(ApplicationEventProcessor.class);
this.requestManagers = requestManagers;
this.metadata = metadata;
}
- public boolean process(final ApplicationEvent event) {
- Objects.requireNonNull(event);
+ /**
+ * Process the events—if any—that were produced by the application thread. It is possible that when processing
+ * an event generates an error. In such cases, the processor will log an exception, but we do not want those
+ * errors to be propagated to the caller.
+ */
+ @Override
+ public void process() {
+ process((event, error) -> { });
+ }
+
+ @Override
+ public void process(ApplicationEvent event) {
switch (event.type()) {
- case NOOP:
- return process((NoopApplicationEvent) event);
case COMMIT:
- return process((CommitApplicationEvent) event);
+ process((CommitApplicationEvent) event);
+ return;
+
case POLL:
- return process((PollApplicationEvent) event);
+ process((PollApplicationEvent) event);
+ return;
+
case FETCH_COMMITTED_OFFSET:
- return process((OffsetFetchApplicationEvent) event);
+ process((OffsetFetchApplicationEvent) event);
+ return;
+
case METADATA_UPDATE:
- return process((NewTopicsMetadataUpdateRequestEvent) event);
+ process((NewTopicsMetadataUpdateRequestEvent) event);
+ return;
+
case ASSIGNMENT_CHANGE:
- return process((AssignmentChangeApplicationEvent) event);
+ process((AssignmentChangeApplicationEvent) event);
+ return;
+
case TOPIC_METADATA:
- return process((TopicMetadataApplicationEvent) event);
+ process((TopicMetadataApplicationEvent) event);
+ return;
+
case LIST_OFFSETS:
- return process((ListOffsetsApplicationEvent) event);
+ process((ListOffsetsApplicationEvent) event);
+ return;
+
case RESET_POSITIONS:
- return processResetPositionsEvent();
+ processResetPositionsEvent();
+ return;
+
case VALIDATE_POSITIONS:
- return processValidatePositionsEvent();
+ processValidatePositionsEvent();
+ return;
+
+ default:
+ log.warn("Application event type " + event.type() + " was not expected");
}
- return false;
}
- /**
- * Processes {@link NoopApplicationEvent} and enqueue a
- * {@link NoopBackgroundEvent}. This is intentionally left here for
- * demonstration purpose.
- *
- * @param event a {@link NoopApplicationEvent}
- */
- private boolean process(final NoopApplicationEvent event) {
- return backgroundEventQueue.add(new NoopBackgroundEvent(event.message()));
+ @Override
+ protected Class<ApplicationEvent> getEventClass() {
+ return ApplicationEvent.class;
}
- private boolean process(final PollApplicationEvent event) {
+ private void process(final PollApplicationEvent event) {
if (!requestManagers.commitRequestManager.isPresent()) {
- return true;
+ return;
}
CommitRequestManager manager = requestManagers.commitRequestManager.get();
manager.updateAutoCommitTimer(event.pollTimeMs());
- return true;
}
- private boolean process(final CommitApplicationEvent event) {
+ private void process(final CommitApplicationEvent event) {
if (!requestManagers.commitRequestManager.isPresent()) {
// Leaving this error handling here, but it is a bit strange as the commit API should enforce the group.id
- // upfront so we should never get to this block.
+ // upfront, so we should never get to this block.
Exception exception = new KafkaException("Unable to commit offset. Most likely because the group.id wasn't set");
event.future().completeExceptionally(exception);
- return false;
+ return;
}
CommitRequestManager manager = requestManagers.commitRequestManager.get();
event.chain(manager.addOffsetCommitRequest(event.offsets()));
- return true;
}
- private boolean process(final OffsetFetchApplicationEvent event) {
+ private void process(final OffsetFetchApplicationEvent event) {
if (!requestManagers.commitRequestManager.isPresent()) {
event.future().completeExceptionally(new KafkaException("Unable to fetch committed " +
"offset because the CommittedRequestManager is not available. Check if group.id was set correctly"));
- return false;
+ return;
}
CommitRequestManager manager = requestManagers.commitRequestManager.get();
event.chain(manager.addOffsetFetchRequest(event.partitions()));
- return true;
}
- private boolean process(final NewTopicsMetadataUpdateRequestEvent event) {
+ private void process(final NewTopicsMetadataUpdateRequestEvent ignored) {
metadata.requestUpdateForNewTopics();
- return true;
}
- private boolean process(final AssignmentChangeApplicationEvent event) {
+ private void process(final AssignmentChangeApplicationEvent event) {
if (!requestManagers.commitRequestManager.isPresent()) {
- return false;
+ return;
}
CommitRequestManager manager = requestManagers.commitRequestManager.get();
manager.updateAutoCommitTimer(event.currentTimeMs());
manager.maybeAutoCommit(event.offsets());
- return true;
}
- private boolean process(final ListOffsetsApplicationEvent event) {
+ private void process(final ListOffsetsApplicationEvent event) {
final CompletableFuture<Map<TopicPartition, OffsetAndTimestamp>> future =
requestManagers.offsetsRequestManager.fetchOffsets(event.timestampsToSearch(),
event.requireTimestamps());
event.chain(future);
- return true;
}
- private boolean processResetPositionsEvent() {
+ private void processResetPositionsEvent() {
requestManagers.offsetsRequestManager.resetPositionsIfNeeded();
- return true;
}
- private boolean processValidatePositionsEvent() {
+ private void processValidatePositionsEvent() {
requestManagers.offsetsRequestManager.validatePositionsIfNeeded();
- return true;
}
- private boolean process(final TopicMetadataApplicationEvent event) {
+ private void process(final TopicMetadataApplicationEvent event) {
final CompletableFuture<Map<String, List<PartitionInfo>>> future =
- this.requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic()));
+ this.requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic()));
event.chain(future);
- return true;
+ }
+
+ /**
+ * Creates a {@link Supplier} for deferred creation during invocation by
+ * {@link ConsumerNetworkThread}.
+ */
+ public static Supplier<ApplicationEventProcessor> supplier(final LogContext logContext,
+ final ConsumerMetadata metadata,
+ final BlockingQueue<ApplicationEvent> applicationEventQueue,
+ final Supplier<RequestManagers> requestManagersSupplier) {
+ return new CachedSupplier<ApplicationEventProcessor>() {
+ @Override
+ protected ApplicationEventProcessor create() {
+ RequestManagers requestManagers = requestManagersSupplier.get();
+ return new ApplicationEventProcessor(
+ logContext,
+ applicationEventQueue,
+ requestManagers,
+ metadata
+ );
+ }
+ };
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java
index b0f7c3454f6..a7dc3e454a7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java
@@ -16,15 +16,17 @@
*/
package org.apache.kafka.clients.consumer.internals.events;
+import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread;
+
import java.util.Objects;
/**
- * This is the abstract definition of the events created by the background thread.
+ * This is the abstract definition of the events created by the {@link ConsumerNetworkThread network thread}.
*/
public abstract class BackgroundEvent {
public enum Type {
- NOOP, ERROR,
+ ERROR,
}
protected final Type type;
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java
new file mode 100644
index 00000000000..cafa426d033
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals.events;
+
+import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread;
+import org.apache.kafka.common.utils.LogContext;
+import org.slf4j.Logger;
+
+import java.util.Objects;
+import java.util.Queue;
+
+/**
+ * An event handler that receives {@link BackgroundEvent background events} from the
+ * {@link ConsumerNetworkThread network thread} which are then made available to the application thread
+ * via the {@link BackgroundEventProcessor}.
+ */
+
+public class BackgroundEventHandler {
+
+ private final Logger log;
+ private final Queue<BackgroundEvent> backgroundEventQueue;
+
+ public BackgroundEventHandler(final LogContext logContext, final Queue<BackgroundEvent> backgroundEventQueue) {
+ this.log = logContext.logger(BackgroundEventHandler.class);
+ this.backgroundEventQueue = backgroundEventQueue;
+ }
+
+ /**
+ * Add a {@link BackgroundEvent} to the handler.
+ *
+ * @param event A {@link BackgroundEvent} created by the {@link ConsumerNetworkThread network thread}
+ */
+ public void add(BackgroundEvent event) {
+ Objects.requireNonNull(event, "BackgroundEvent provided to add must be non-null");
+ log.trace("Enqueued event: {}", event);
+ backgroundEventQueue.add(event);
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java
new file mode 100644
index 00000000000..cafd4fba492
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals.events;
+
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.utils.LogContext;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * An {@link EventProcessor} that is created and executes in the application thread for the purpose of processing
+ * {@link BackgroundEvent background events} generated by the {@link ConsumerNetworkThread network thread}.
+ * Those events are generally of two types:
+ *
+ * <ul>
+ * <li>Errors that occur in the network thread that need to be propagated to the application thread</li>
+ * <li>{@link ConsumerRebalanceListener} callbacks that are to be executed on the application thread</li>
+ * </ul>
+ */
+public class BackgroundEventProcessor extends EventProcessor<BackgroundEvent> {
+
+ public BackgroundEventProcessor(final LogContext logContext,
+ final BlockingQueue<BackgroundEvent> backgroundEventQueue) {
+ super(logContext, backgroundEventQueue);
+ }
+
+ /**
+ * Process the events—if any—that were produced by the {@link ConsumerNetworkThread network thread}.
+ * It is possible that {@link ErrorBackgroundEvent an error} could occur when processing the events.
+ * In such cases, the processor will take a reference to the first error, continue to process the
+ * remaining events, and then throw the first error that occurred.
+ */
+ @Override
+ public void process() {
+ AtomicReference<KafkaException> firstError = new AtomicReference<>();
+ process((event, error) -> firstError.compareAndSet(null, error));
+
+ if (firstError.get() != null)
+ throw firstError.get();
+ }
+
+ @Override
+ public void process(final BackgroundEvent event) {
+ if (event.type() == BackgroundEvent.Type.ERROR)
+ process((ErrorBackgroundEvent) event);
+ else
+ throw new IllegalArgumentException("Background event type " + event.type() + " was not expected");
+ }
+
+ @Override
+ protected Class<BackgroundEvent> getEventClass() {
+ return BackgroundEvent.class;
+ }
+
+ private void process(final ErrorBackgroundEvent event) {
+ throw event.error();
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java
index 8146d9583ae..365c620e0c0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java
@@ -27,7 +27,7 @@ import java.util.concurrent.CompletableFuture;
*
* @param <T>
*/
-public abstract class CompletableApplicationEvent<T> extends ApplicationEvent {
+public abstract class CompletableApplicationEvent<T> extends ApplicationEvent implements CompletableEvent<T> {
private final CompletableFuture<T> future;
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java
similarity index 68%
copy from clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java
copy to clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java
index 4655f5ef704..8fdcc20fa83 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java
@@ -14,15 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kafka.clients.consumer.internals;
+package org.apache.kafka.clients.consumer.internals.events;
-import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult;
+import java.util.concurrent.CompletableFuture;
-/**
- * {@code PollResult} consist of {@code UnsentRequest} if there are requests to send; otherwise, return the time till
- * the next poll event.
- */
-public interface RequestManager {
- PollResult poll(long currentTimeMs);
+public interface CompletableEvent<T> {
+
+ CompletableFuture<T> future();
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java
index 74d35983291..2945f22986b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java
@@ -16,16 +16,18 @@
*/
package org.apache.kafka.clients.consumer.internals.events;
+import org.apache.kafka.common.KafkaException;
+
public class ErrorBackgroundEvent extends BackgroundEvent {
- private final Throwable error;
+ private final RuntimeException error;
- public ErrorBackgroundEvent(Throwable error) {
+ public ErrorBackgroundEvent(Throwable t) {
super(Type.ERROR);
- this.error = error;
+ this.error = t instanceof RuntimeException ? (RuntimeException) t : new KafkaException(t);
}
- public Throwable error() {
+ public RuntimeException error() {
return error;
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java
deleted file mode 100644
index 75075fc2779..00000000000
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals.events;
-
-import org.apache.kafka.common.utils.Timer;
-
-import java.io.Closeable;
-import java.util.Optional;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This class interfaces with the KafkaConsumer and the background thread. It allows the caller to enqueue events via
- * the {@code add()} method and to retrieve events via the {@code poll()} method.
- */
-public interface EventHandler extends Closeable {
- /**
- * Retrieves and removes a {@link BackgroundEvent}. Returns an empty Optional instance if there is nothing.
- * @return an Optional of {@link BackgroundEvent} if the value is present. Otherwise, an empty Optional.
- */
- Optional<BackgroundEvent> poll();
-
- /**
- * Check whether there are pending {@code BackgroundEvent} await to be consumed.
- * @return true if there are no pending event
- */
- boolean isEmpty();
-
- /**
- * Add an {@link ApplicationEvent} to the handler. The method returns true upon successful add; otherwise returns
- * false.
- * @param event An {@link ApplicationEvent} created by the polling thread.
- * @return true upon successful add.
- */
- boolean add(ApplicationEvent event);
-
- /**
- * Add a {@link CompletableApplicationEvent} to the handler. The method blocks waiting for the result, and will
- * return the result value upon successful completion; otherwise throws an error.
- *
- * <p/>
- *
- * See {@link CompletableApplicationEvent#get(Timer)} and {@link Future#get(long, TimeUnit)} for more details.
- *
- * @param event A {@link CompletableApplicationEvent} created by the polling thread.
- * @param timer Timer for which to wait for the event to complete
- * @return Value that is the result of the event
- * @param <T> Type of return value of the event
- */
- <T> T addAndGet(final CompletableApplicationEvent<T> event, final Timer timer);
-}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java
new file mode 100644
index 00000000000..06e7ec28dd7
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals.events;
+
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.internals.IdempotentCloser;
+import org.apache.kafka.common.utils.LogContext;
+import org.slf4j.Logger;
+
+import java.io.Closeable;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * An {@link EventProcessor} is the means by which events <em>produced</em> by thread <em>A</em> are
+ * <em>processed</em> by thread <em>B</em>. By definition, threads <em>A</em> and <em>B</em> run in parallel to
+ * each other, so a mechanism is needed with which to receive and process the events from the other thread. That
+ * communication channel is formed around {@link BlockingQueue a shared queue} into which thread <em>A</em>
+ * enqueues events and thread <em>B</em> reads and processes those events.
+ */
+public abstract class EventProcessor<T> implements Closeable {
+
+ private final Logger log;
+ private final BlockingQueue<T> eventQueue;
+ private final IdempotentCloser closer;
+
+ protected EventProcessor(final LogContext logContext, final BlockingQueue<T> eventQueue) {
+ this.log = logContext.logger(EventProcessor.class);
+ this.eventQueue = eventQueue;
+ this.closer = new IdempotentCloser();
+ }
+
+ public abstract void process();
+
+ public abstract void process(T event);
+
+ @Override
+ public void close() {
+ closer.close(this::closeInternal, () -> log.warn("The event processor was already closed"));
+ }
+
+ protected abstract Class<T> getEventClass();
+
+ protected interface ProcessErrorHandler<T> {
+
+ void onError(T event, KafkaException error);
+ }
+
+ /**
+ * Drains all available events from the queue, and then processes them in order. If any errors are thrown while
+ * processing the individual events, these are submitted to the given {@link ProcessErrorHandler}.
+ */
+ protected void process(ProcessErrorHandler<T> processErrorHandler) {
+ String eventClassName = getEventClass().getSimpleName();
+ closer.assertOpen(() -> String.format("The processor was previously closed, so no further %s processing can occur", eventClassName));
+
+ List<T> events = drain();
+
+ try {
+ log.debug("Starting processing of {} {}(s)", events.size(), eventClassName);
+
+ for (T event : events) {
+ try {
+ Objects.requireNonNull(event, () -> String.format("Attempted to process a null %s", eventClassName));
+ log.debug("Consuming {}: {}", eventClassName, event);
+ process(event);
+ } catch (Throwable t) {
+ log.warn("An error occurred when processing the {}: {}", eventClassName, t.getMessage(), t);
+
+ KafkaException error;
+
+ if (t instanceof KafkaException)
+ error = (KafkaException) t;
+ else
+ error = new KafkaException(t);
+
+ processErrorHandler.onError(event, error);
+ }
+ }
+ } finally {
+ log.debug("Completed processing of {} {}(s)", events.size(), eventClassName);
+ }
+ }
+
+ /**
+ * It is possible for the consumer to close before complete processing all the events in the queue. In
+ * this case, we need to throw an exception to notify the user the consumer is closed.
+ */
+ private void closeInternal() {
+ String eventClassName = getEventClass().getSimpleName();
+ log.trace("Closing event processor for {}", eventClassName);
+ List<T> incompleteEvents = drain();
+
+ if (incompleteEvents.isEmpty())
+ return;
+
+ KafkaException exception = new KafkaException("The consumer is closed");
+
+ // Check each of the events and if it has a Future that is incomplete, complete it exceptionally.
+ incompleteEvents
+ .stream()
+ .filter(e -> e instanceof CompletableEvent)
+ .map(e -> ((CompletableEvent<?>) e).future())
+ .filter(f -> !f.isDone())
+ .forEach(f -> {
+ log.debug("Completing {} with exception {}", f, exception.getMessage());
+ f.completeExceptionally(exception);
+ });
+
+ log.debug("Discarding {} {}s because the consumer is closing", incompleteEvents.size(), eventClassName);
+ }
+
+ /**
+ * Moves all the events from the queue to the returned list.
+ */
+ private List<T> drain() {
+ LinkedList<T> events = new LinkedList<>();
+ eventQueue.drainTo(events);
+ return events;
+ }
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java
deleted file mode 100644
index 22817fb2bae..00000000000
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals.events;
-
-import java.util.Objects;
-
-/**
- * The event is a no-op, but is intentionally left here for demonstration and test purposes.
- */
-public class NoopApplicationEvent extends ApplicationEvent {
-
- private final String message;
-
- public NoopApplicationEvent(final String message) {
- super(Type.NOOP);
- this.message = Objects.requireNonNull(message);
- }
-
- public String message() {
- return message;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
-
- NoopApplicationEvent that = (NoopApplicationEvent) o;
-
- return message.equals(that.message);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + message.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "NoopApplicationEvent{" +
- toStringBase() +
- ",message='" + message + '\'' +
- '}';
- }
-}
\ No newline at end of file
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java
deleted file mode 100644
index c1cbcc253a9..00000000000
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals.events;
-
-import java.util.Objects;
-
-/**
- * No-op event. Intentionally left it here for demonstration purpose.
- */
-public class NoopBackgroundEvent extends BackgroundEvent {
-
- private final String message;
-
- public NoopBackgroundEvent(final String message) {
- super(Type.NOOP);
- this.message = Objects.requireNonNull(message);
- }
-
- public String message() {
- return message;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
-
- NoopBackgroundEvent that = (NoopBackgroundEvent) o;
-
- return message.equals(that.message);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + message.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "NoopBackgroundEvent{" +
- toStringBase() +
- ", message='" + message + '\'' +
- '}';
- }
-}
\ No newline at end of file
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index f61d4547d8b..6a46b453191 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -2674,7 +2674,7 @@ public class KafkaConsumerTest {
}
IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED;
FetchMetricsManager metricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics);
- FetchConfig<String, String> fetchConfig = new FetchConfig<>(
+ FetchConfig fetchConfig = new FetchConfig(
minBytes,
maxBytes,
maxWaitMs,
@@ -2682,7 +2682,6 @@ public class KafkaConsumerTest {
maxPollRecords,
checkCrcs,
CommonClientConfigs.DEFAULT_CLIENT_RACK,
- new Deserializers<>(keyDeserializer, deserializer),
isolationLevel);
Fetcher<String, String> fetcher = new Fetcher<>(
loggerFactory,
@@ -2690,6 +2689,7 @@ public class KafkaConsumerTest {
metadata,
subscription,
fetchConfig,
+ new Deserializers<>(keyDeserializer, deserializer),
metricsManager,
time);
OffsetFetcher offsetFetcher = new OffsetFetcher(loggerFactory,
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
index c8a2e1ee945..bb58c9423c5 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
@@ -55,6 +55,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTER
import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG;
import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -76,7 +77,7 @@ public class CommitRequestManagerTest {
this.time = new MockTime(0);
this.subscriptionState = mock(SubscriptionState.class);
this.coordinatorRequestManager = mock(CoordinatorRequestManager.class);
- this.groupState = new GroupState("group-1", Optional.empty());
+ this.groupState = new GroupState(DEFAULT_GROUP_ID, Optional.empty());
this.props = new Properties();
this.props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java
index de97ee40de2..2c39f4411fe 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java
@@ -34,7 +34,6 @@ import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.record.Records;
import org.apache.kafka.common.record.SimpleRecord;
import org.apache.kafka.common.record.TimestampType;
-import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.UUIDDeserializer;
import org.apache.kafka.common.serialization.UUIDSerializer;
@@ -67,23 +66,22 @@ public class CompletedFetchTest {
FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData()
.setRecords(newRecords(startingOffset, numRecords, fetchOffset));
- FetchConfig<String, String> fetchConfig = newFetchConfig(new StringDeserializer(),
- new StringDeserializer(),
- IsolationLevel.READ_UNCOMMITTED,
- true);
+ Deserializers<String, String> deserializers = newStringDeserializers();
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true);
+
CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData);
- List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, 10);
+ List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(10, records.size());
ConsumerRecord<String, String> record = records.get(0);
assertEquals(10, record.offset());
- records = completedFetch.fetchRecords(fetchConfig, 10);
+ records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(1, records.size());
record = records.get(0);
assertEquals(20, record.offset());
- records = completedFetch.fetchRecords(fetchConfig, 10);
+ records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(0, records.size());
}
@@ -96,21 +94,15 @@ public class CompletedFetchTest {
.setRecords(rawRecords)
.setAbortedTransactions(newAbortedTransactions());
- try (final StringDeserializer deserializer = new StringDeserializer()) {
- FetchConfig<String, String> fetchConfig = newFetchConfig(deserializer,
- deserializer,
- IsolationLevel.READ_COMMITTED,
- true);
+ try (final Deserializers<String, String> deserializers = newStringDeserializers()) {
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true);
CompletedFetch completedFetch = newCompletedFetch(0, partitionData);
- List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, 10);
+ List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(0, records.size());
- fetchConfig = newFetchConfig(deserializer,
- deserializer,
- IsolationLevel.READ_UNCOMMITTED,
- true);
+ fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true);
completedFetch = newCompletedFetch(0, partitionData);
- records = completedFetch.fetchRecords(fetchConfig, 10);
+ records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(numRecords, records.size());
}
}
@@ -122,12 +114,9 @@ public class CompletedFetchTest {
FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData()
.setRecords(rawRecords);
CompletedFetch completedFetch = newCompletedFetch(0, partitionData);
- try (final StringDeserializer deserializer = new StringDeserializer()) {
- FetchConfig<String, String> fetchConfig = newFetchConfig(deserializer,
- deserializer,
- IsolationLevel.READ_COMMITTED,
- true);
- List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, 10);
+ try (final Deserializers<String, String> deserializers = newStringDeserializers()) {
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true);
+ List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(10, records.size());
}
}
@@ -140,14 +129,13 @@ public class CompletedFetchTest {
FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData()
.setRecords(newRecords(startingOffset, numRecords, fetchOffset));
- CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData);
- FetchConfig<String, String> fetchConfig = newFetchConfig(new StringDeserializer(),
- new StringDeserializer(),
- IsolationLevel.READ_UNCOMMITTED,
- true);
+ try (final Deserializers<String, String> deserializers = newStringDeserializers()) {
+ CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData);
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true);
- List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, -10);
- assertEquals(0, records.size());
+ List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, deserializers, -10);
+ assertEquals(0, records.size());
+ }
}
@Test
@@ -159,13 +147,9 @@ public class CompletedFetchTest {
.setLogStartOffset(0);
CompletedFetch completedFetch = newCompletedFetch(1, partitionData);
- try (final StringDeserializer deserializer = new StringDeserializer()) {
- FetchConfig<String, String> fetchConfig = newFetchConfig(deserializer,
- deserializer,
- IsolationLevel.READ_UNCOMMITTED,
- true);
-
- List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, 10);
+ try (final Deserializers<String, String> deserializers = newStringDeserializers()) {
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true);
+ List<ConsumerRecord<String, String>> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10);
assertEquals(0, records.size());
}
}
@@ -174,8 +158,7 @@ public class CompletedFetchTest {
public void testCorruptedMessage() {
// Create one good record and then one "corrupted" record.
try (final MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0);
- final UUIDSerializer serializer = new UUIDSerializer();
- final UUIDDeserializer deserializer = new UUIDDeserializer()) {
+ final UUIDSerializer serializer = new UUIDSerializer()) {
builder.append(new SimpleRecord(serializer.serialize(TOPIC_NAME, UUID.randomUUID())));
builder.append(0L, "key".getBytes(), "value".getBytes());
Records records = builder.build();
@@ -187,16 +170,15 @@ public class CompletedFetchTest {
.setLogStartOffset(0)
.setRecords(records);
- FetchConfig<UUID, UUID> fetchConfig = newFetchConfig(deserializer,
- deserializer,
- IsolationLevel.READ_COMMITTED,
- false);
- CompletedFetch completedFetch = newCompletedFetch(0, partitionData);
+ try (final Deserializers<UUID, UUID> deserializers = newUuidDeserializers()) {
+ FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, false);
+ CompletedFetch completedFetch = newCompletedFetch(0, partitionData);
- completedFetch.fetchRecords(fetchConfig, 10);
+ completedFetch.fetchRecords(fetchConfig, deserializers, 10);
- assertThrows(RecordDeserializationException.class,
- () -> completedFetch.fetchRecords(fetchConfig, 10));
+ assertThrows(RecordDeserializationException.class,
+ () -> completedFetch.fetchRecords(fetchConfig, deserializers, 10));
+ }
}
}
@@ -219,11 +201,16 @@ public class CompletedFetchTest {
ApiKeys.FETCH.latestVersion());
}
- private static <K, V> FetchConfig<K, V> newFetchConfig(Deserializer<K> keyDeserializer,
- Deserializer<V> valueDeserializer,
- IsolationLevel isolationLevel,
- boolean checkCrcs) {
- return new FetchConfig<>(
+ private static Deserializers<UUID, UUID> newUuidDeserializers() {
+ return new Deserializers<>(new UUIDDeserializer(), new UUIDDeserializer());
+ }
+
+ private static Deserializers<String, String> newStringDeserializers() {
+ return new Deserializers<>(new StringDeserializer(), new StringDeserializer());
+ }
+
+ private static FetchConfig newFetchConfig(IsolationLevel isolationLevel, boolean checkCrcs) {
+ return new FetchConfig(
ConsumerConfig.DEFAULT_FETCH_MIN_BYTES,
ConsumerConfig.DEFAULT_FETCH_MAX_BYTES,
ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS,
@@ -231,7 +218,6 @@ public class CompletedFetchTest {
ConsumerConfig.DEFAULT_MAX_POLL_RECORDS,
checkCrcs,
ConsumerConfig.DEFAULT_CLIENT_RACK,
- new Deserializers<>(keyDeserializer, valueDeserializer),
isolationLevel
);
}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 43ceca65bb7..cc0b1294fc2 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -3082,7 +3082,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L));
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertTrue(subscriptions.hasAllFetchPositions());
@@ -3103,7 +3103,7 @@ public abstract class ConsumerCoordinatorTest {
// Load offsets from previous epoch
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L, Optional.of(3)));
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
// Offset gets loaded, but requires validation
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
@@ -3155,7 +3155,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
client.prepareResponse(offsetFetchResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS, Collections.emptyMap()));
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L));
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertTrue(subscriptions.hasAllFetchPositions());
@@ -3170,7 +3170,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
client.prepareResponse(offsetFetchResponse(Errors.GROUP_AUTHORIZATION_FAILED, Collections.emptyMap()));
try {
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
fail("Expected group authorization error");
} catch (GroupAuthorizationException e) {
assertEquals(groupId, e.groupId());
@@ -3186,9 +3186,9 @@ public abstract class ConsumerCoordinatorTest {
client.prepareResponse(offsetFetchResponse(t1p, Errors.UNSTABLE_OFFSET_COMMIT, "", -1L));
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L));
assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions());
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(0L));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(0L));
assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions());
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(0L));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(0L));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertTrue(subscriptions.hasAllFetchPositions());
@@ -3202,7 +3202,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
client.prepareResponse(offsetFetchResponse(t1p, Errors.UNKNOWN_TOPIC_OR_PARTITION, "", 100L));
- assertThrows(KafkaException.class, () -> coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)));
+ assertThrows(KafkaException.class, () -> coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)));
}
@Test
@@ -3214,7 +3214,7 @@ public abstract class ConsumerCoordinatorTest {
client.prepareResponse(offsetFetchResponse(Errors.NOT_COORDINATOR, Collections.emptyMap()));
client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L));
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertTrue(subscriptions.hasAllFetchPositions());
@@ -3228,7 +3228,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", -1L));
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions());
assertEquals(Collections.emptySet(), subscriptions.partitionsNeedingReset(time.milliseconds()));
@@ -3242,7 +3242,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
subscriptions.seek(t1p, 500L);
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertTrue(subscriptions.hasAllFetchPositions());
@@ -3256,7 +3256,7 @@ public abstract class ConsumerCoordinatorTest {
subscriptions.assignFromUser(singleton(t1p));
subscriptions.requestOffsetReset(t1p, OffsetResetStrategy.EARLIEST);
- coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
+ coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE));
assertEquals(Collections.emptySet(), subscriptions.initializingPartitions());
assertFalse(subscriptions.hasAllFetchPositions());
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java
new file mode 100644
index 00000000000..812999b3ca3
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import org.apache.kafka.clients.MockClient;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
+import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent;
+import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.message.FindCoordinatorRequestData;
+import org.apache.kafka.common.requests.FindCoordinatorRequest;
+import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.RequestTestUtils;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.test.TestCondition;
+import org.apache.kafka.test.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REQUEST_TIMEOUT_MS;
+import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ClassDataAbstractionCoupling")
+public class ConsumerNetworkThreadTest {
+
+ private ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder testBuilder;
+ private Time time;
+ private ConsumerMetadata metadata;
+ private NetworkClientDelegate networkClient;
+ private BlockingQueue<ApplicationEvent> applicationEventsQueue;
+ private ApplicationEventProcessor applicationEventProcessor;
+ private OffsetsRequestManager offsetsRequestManager;
+ private CommitRequestManager commitManager;
+ private ConsumerNetworkThread consumerNetworkThread;
+ private MockClient client;
+
+ @BeforeEach
+ public void setup() {
+ testBuilder = new ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder();
+ time = testBuilder.time;
+ metadata = testBuilder.metadata;
+ networkClient = testBuilder.networkClientDelegate;
+ client = testBuilder.client;
+ applicationEventsQueue = testBuilder.applicationEventQueue;
+ applicationEventProcessor = testBuilder.applicationEventProcessor;
+ commitManager = testBuilder.commitRequestManager.orElseThrow(IllegalStateException::new);
+ offsetsRequestManager = testBuilder.offsetsRequestManager;
+ consumerNetworkThread = testBuilder.consumerNetworkThread;
+ consumerNetworkThread.initializeResources();
+ }
+
+ @AfterEach
+ public void tearDown() {
+ if (testBuilder != null)
+ testBuilder.close();
+ }
+
+ @Test
+ public void testStartupAndTearDown() throws InterruptedException {
+ // The consumer is closed in ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder.close()
+ // which is called from tearDown().
+ consumerNetworkThread.start();
+
+ TestCondition isStarted = () -> consumerNetworkThread.isRunning();
+ TestCondition isClosed = () -> !(consumerNetworkThread.isRunning() || consumerNetworkThread.isAlive());
+
+ // There's a nonzero amount of time between starting the thread and having it
+ // begin to execute our code. Wait for a bit before checking...
+ TestUtils.waitForCondition(isStarted,
+ "The consumer network thread did not start within " + DEFAULT_MAX_WAIT_MS + " ms");
+
+ consumerNetworkThread.close(Duration.ofMillis(DEFAULT_MAX_WAIT_MS));
+
+ TestUtils.waitForCondition(isClosed,
+ "The consumer network thread did not stop within " + DEFAULT_MAX_WAIT_MS + " ms");
+ }
+
+ @Test
+ public void testApplicationEvent() {
+ ApplicationEvent e = new CommitApplicationEvent(new HashMap<>());
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor, times(1)).process(e);
+ }
+
+ @Test
+ public void testMetadataUpdateEvent() {
+ ApplicationEvent e = new NewTopicsMetadataUpdateRequestEvent();
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(metadata).requestUpdateForNewTopics();
+ }
+
+ @Test
+ public void testCommitEvent() {
+ ApplicationEvent e = new CommitApplicationEvent(new HashMap<>());
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(CommitApplicationEvent.class));
+ }
+
+ @Test
+ public void testListOffsetsEventIsProcessed() {
+ Map<TopicPartition, Long> timestamps = Collections.singletonMap(new TopicPartition("topic1", 1), 5L);
+ ApplicationEvent e = new ListOffsetsApplicationEvent(timestamps, true);
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(ListOffsetsApplicationEvent.class));
+ assertTrue(applicationEventsQueue.isEmpty());
+ }
+
+ @Test
+ public void testResetPositionsEventIsProcessed() {
+ ResetPositionsApplicationEvent e = new ResetPositionsApplicationEvent();
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class));
+ assertTrue(applicationEventsQueue.isEmpty());
+ }
+
+ @Test
+ public void testResetPositionsProcessFailureIsIgnored() {
+ doThrow(new NullPointerException()).when(offsetsRequestManager).resetPositionsIfNeeded();
+
+ ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent();
+ applicationEventsQueue.add(event);
+ assertDoesNotThrow(() -> consumerNetworkThread.runOnce());
+
+ verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class));
+ }
+
+ @Test
+ public void testValidatePositionsEventIsProcessed() {
+ ValidatePositionsApplicationEvent e = new ValidatePositionsApplicationEvent();
+ applicationEventsQueue.add(e);
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class));
+ assertTrue(applicationEventsQueue.isEmpty());
+ }
+
+ @Test
+ public void testAssignmentChangeEvent() {
+ HashMap<TopicPartition, OffsetAndMetadata> offset = mockTopicPartitionOffset();
+
+ final long currentTimeMs = time.milliseconds();
+ ApplicationEvent e = new AssignmentChangeApplicationEvent(offset, currentTimeMs);
+ applicationEventsQueue.add(e);
+
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(AssignmentChangeApplicationEvent.class));
+ verify(networkClient, times(1)).poll(anyLong(), anyLong());
+ verify(commitManager, times(1)).updateAutoCommitTimer(currentTimeMs);
+ verify(commitManager, times(1)).maybeAutoCommit(offset);
+ }
+
+ @Test
+ void testFetchTopicMetadata() {
+ applicationEventsQueue.add(new TopicMetadataApplicationEvent("topic"));
+ consumerNetworkThread.runOnce();
+ verify(applicationEventProcessor).process(any(TopicMetadataApplicationEvent.class));
+ }
+
+ @Test
+ void testPollResultTimer() {
+ NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest(
+ new FindCoordinatorRequest.Builder(
+ new FindCoordinatorRequestData()
+ .setKeyType(FindCoordinatorRequest.CoordinatorType.TRANSACTION.id())
+ .setKey("foobar")),
+ Optional.empty());
+ req.setTimer(time, DEFAULT_REQUEST_TIMEOUT_MS);
+
+ // purposely setting a non-MAX time to ensure it is returning Long.MAX_VALUE upon success
+ NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult(
+ 10,
+ Collections.singletonList(req));
+ assertEquals(10, networkClient.addAll(success));
+
+ NetworkClientDelegate.PollResult failure = new NetworkClientDelegate.PollResult(
+ 10,
+ new ArrayList<>());
+ assertEquals(10, networkClient.addAll(failure));
+ }
+
+ @Test
+ void testRequestManagersArePolledOnce() {
+ consumerNetworkThread.runOnce();
+ testBuilder.requestManagers.entries().forEach(rmo -> rmo.ifPresent(rm -> verify(rm, times(1)).poll(anyLong())));
+ verify(networkClient, times(1)).poll(anyLong(), anyLong());
+ }
+
+ @Test
+ void testEnsureMetadataUpdateOnPoll() {
+ MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(2, Collections.emptyMap());
+ client.prepareMetadataUpdate(metadataResponse);
+ metadata.requestUpdate(false);
+ consumerNetworkThread.runOnce();
+ verify(metadata, times(1)).updateWithCurrentRequestVersion(eq(metadataResponse), eq(false), anyLong());
+ }
+
+ @Test
+ void testEnsureEventsAreCompleted() {
+ CompletableApplicationEvent<Void> event1 = spy(new CommitApplicationEvent(Collections.emptyMap()));
+ ApplicationEvent event2 = new CommitApplicationEvent(Collections.emptyMap());
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ when(event1.future()).thenReturn(future);
+ applicationEventsQueue.add(event1);
+ applicationEventsQueue.add(event2);
+ assertFalse(future.isDone());
+ assertFalse(applicationEventsQueue.isEmpty());
+
+ consumerNetworkThread.cleanup();
+ assertTrue(future.isCompletedExceptionally());
+ assertTrue(applicationEventsQueue.isEmpty());
+ }
+
+ private HashMap<TopicPartition, OffsetAndMetadata> mockTopicPartitionOffset() {
+ final TopicPartition t0 = new TopicPartition("t0", 2);
+ final TopicPartition t1 = new TopicPartition("t0", 3);
+ HashMap<TopicPartition, OffsetAndMetadata> topicPartitionOffsets = new HashMap<>();
+ topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L));
+ topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L));
+ return topicPartitionOffsets;
+ }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java
new file mode 100644
index 00000000000..894f8305d02
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.consumer.internals;
+
+import org.apache.kafka.clients.ApiVersions;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.GroupRebalanceConfig;
+import org.apache.kafka.clients.MockClient;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor;
+import org.apache.kafka.common.internals.ClusterResourceListeners;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.RequestTestUtils;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.utils.LogContext;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+
+import java.io.Closeable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics;
+import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState;
+import static org.apache.kafka.common.utils.Utils.closeQuietly;
+import static org.mockito.Mockito.spy;
+
+@SuppressWarnings("ClassDataAbstractionCoupling")
+public class ConsumerTestBuilder implements Closeable {
+
+ static final long DEFAULT_RETRY_BACKOFF_MS = 80;
+ static final long DEFAULT_RETRY_BACKOFF_MAX_MS = 1000;
+ static final int DEFAULT_REQUEST_TIMEOUT_MS = 500;
+ static final int DEFAULT_MAX_POLL_INTERVAL_MS = 10000;
+ static final String DEFAULT_GROUP_INSTANCE_ID = "group-instance-id";
+ static final String DEFAULT_GROUP_ID = "group-id";
+ static final int DEFAULT_HEARTBEAT_INTERVAL_MS = 1000;
+ static final double DEFAULT_HEARTBEAT_JITTER_MS = 0.0;
+
+ final LogContext logContext = new LogContext();
+ final Time time = new MockTime(0);
+ public final BlockingQueue<ApplicationEvent> applicationEventQueue;
+ public final BlockingQueue<BackgroundEvent> backgroundEventQueue;
+ final ConsumerConfig config;
+ final long retryBackoffMs;
+ final SubscriptionState subscriptions;
+ final ConsumerMetadata metadata;
+ final FetchConfig fetchConfig;
+ final FetchBuffer fetchBuffer;
+ final Metrics metrics;
+ final FetchMetricsManager metricsManager;
+ final NetworkClientDelegate networkClientDelegate;
+ final OffsetsRequestManager offsetsRequestManager;
+ final Optional<CoordinatorRequestManager> coordinatorRequestManager;
+ final Optional<CommitRequestManager> commitRequestManager;
+ final Optional<HeartbeatRequestManager> heartbeatRequestManager;
+ final Optional<MembershipManager> membershipManager;
+ final Optional<HeartbeatRequestManager.HeartbeatRequestState> heartbeatRequestState;
+ final TopicMetadataRequestManager topicMetadataRequestManager;
+ final FetchRequestManager fetchRequestManager;
+ final RequestManagers requestManagers;
+ public final ApplicationEventProcessor applicationEventProcessor;
+ public final BackgroundEventProcessor backgroundEventProcessor;
+ public final BackgroundEventHandler backgroundEventHandler;
+ final MockClient client;
+ final Optional<GroupInformation> groupInfo;
+
+ public ConsumerTestBuilder() {
+ this(Optional.empty());
+ }
+
+ public ConsumerTestBuilder(Optional<GroupInformation> groupInfo) {
+ this.groupInfo = groupInfo;
+ this.applicationEventQueue = new LinkedBlockingQueue<>();
+ this.backgroundEventQueue = new LinkedBlockingQueue<>();
+ this.backgroundEventHandler = spy(new BackgroundEventHandler(logContext, backgroundEventQueue));
+ GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(
+ 100,
+ DEFAULT_MAX_POLL_INTERVAL_MS,
+ DEFAULT_HEARTBEAT_INTERVAL_MS,
+ groupInfo.map(gi -> gi.groupState.groupId).orElse(null),
+ groupInfo.flatMap(gi -> gi.groupState.groupInstanceId),
+ DEFAULT_RETRY_BACKOFF_MS,
+ DEFAULT_RETRY_BACKOFF_MAX_MS,
+ true);
+ GroupState groupState = new GroupState(groupRebalanceConfig);
+ ApiVersions apiVersions = new ApiVersions();
+
+ Properties properties = new Properties();
+ properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+ properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+ properties.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, DEFAULT_RETRY_BACKOFF_MS);
+ properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_REQUEST_TIMEOUT_MS);
+ properties.put(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG, DEFAULT_MAX_POLL_INTERVAL_MS);
+
+ groupInfo.ifPresent(gi -> {
+ properties.put(GROUP_ID_CONFIG, gi.groupState.groupId);
+ gi.groupState.groupInstanceId.ifPresent(groupInstanceId -> properties.put(GROUP_INSTANCE_ID_CONFIG, groupInstanceId));
+ });
+
+ this.config = new ConsumerConfig(properties);
+
+ this.fetchConfig = new FetchConfig(config);
+ this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG);
+ final long requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG);
+ this.metrics = createMetrics(config, time);
+
+ this.subscriptions = spy(createSubscriptionState(config, logContext));
+ this.metadata = spy(new ConsumerMetadata(config, subscriptions, logContext, new ClusterResourceListeners()));
+ this.metricsManager = createFetchMetricsManager(metrics);
+
+ this.client = new MockClient(time, metadata);
+ MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(1, new HashMap<String, Integer>() {
+ {
+ String topic1 = "test1";
+ put(topic1, 1);
+ String topic2 = "test2";
+ put(topic2, 1);
+ }
+ });
+ this.client.updateMetadata(metadataResponse);
+
+ this.networkClientDelegate = spy(new NetworkClientDelegate(time,
+ config,
+ logContext,
+ client));
+ this.offsetsRequestManager = spy(new OffsetsRequestManager(subscriptions,
+ metadata,
+ fetchConfig.isolationLevel,
+ time,
+ retryBackoffMs,
+ requestTimeoutMs,
+ apiVersions,
+ networkClientDelegate,
+ backgroundEventHandler,
+ logContext));
+
+ if (groupInfo.isPresent()) {
+ GroupInformation gi = groupInfo.get();
+ CoordinatorRequestManager coordinator = spy(new CoordinatorRequestManager(
+ time,
+ logContext,
+ DEFAULT_RETRY_BACKOFF_MS,
+ DEFAULT_RETRY_BACKOFF_MAX_MS,
+ backgroundEventHandler,
+ gi.groupState.groupId
+ ));
+ CommitRequestManager commit = spy(new CommitRequestManager(time,
+ logContext,
+ subscriptions,
+ config,
+ coordinator,
+ groupState));
+ MembershipManager mm = spy(
+ new MembershipManagerImpl(
+ gi.groupState.groupId,
+ gi.groupState.groupInstanceId.orElse(null),
+ null,
+ logContext
+ )
+ );
+ HeartbeatRequestManager.HeartbeatRequestState state = spy(new HeartbeatRequestManager.HeartbeatRequestState(logContext,
+ time,
+ gi.heartbeatIntervalMs,
+ retryBackoffMs,
+ DEFAULT_RETRY_BACKOFF_MAX_MS,
+ gi.heartbeatJitterMs));
+ HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager(
+ logContext,
+ time,
+ config,
+ coordinator,
+ subscriptions,
+ mm,
+ state,
+ backgroundEventHandler));
+
+ this.coordinatorRequestManager = Optional.of(coordinator);
+ this.commitRequestManager = Optional.of(commit);
+ this.heartbeatRequestManager = Optional.of(heartbeat);
+ this.heartbeatRequestState = Optional.of(state);
+ this.membershipManager = Optional.of(mm);
+ } else {
+ this.coordinatorRequestManager = Optional.empty();
+ this.commitRequestManager = Optional.empty();
+ this.heartbeatRequestManager = Optional.empty();
+ this.heartbeatRequestState = Optional.empty();
+ this.membershipManager = Optional.empty();
+ }
+
+ this.fetchBuffer = new FetchBuffer(logContext);
+ this.fetchRequestManager = spy(new FetchRequestManager(logContext,
+ time,
+ metadata,
+ subscriptions,
+ fetchConfig,
+ fetchBuffer,
+ metricsManager,
+ networkClientDelegate));
+ this.topicMetadataRequestManager = spy(new TopicMetadataRequestManager(logContext,
+ config));
+ this.requestManagers = new RequestManagers(logContext,
+ offsetsRequestManager,
+ topicMetadataRequestManager,
+ fetchRequestManager,
+ coordinatorRequestManager,
+ commitRequestManager,
+ heartbeatRequestManager);
+ this.applicationEventProcessor = spy(new ApplicationEventProcessor(
+ logContext,
+ applicationEventQueue,
+ requestManagers,
+ metadata)
+ );
+ this.backgroundEventProcessor = spy(new BackgroundEventProcessor(logContext, backgroundEventQueue));
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(requestManagers, RequestManagers.class.getSimpleName());
+ closeQuietly(applicationEventProcessor, ApplicationEventProcessor.class.getSimpleName());
+ closeQuietly(backgroundEventProcessor, BackgroundEventProcessor.class.getSimpleName());
+ }
+
+ public static class ConsumerNetworkThreadTestBuilder extends ConsumerTestBuilder {
+
+ final ConsumerNetworkThread consumerNetworkThread;
+
+ public ConsumerNetworkThreadTestBuilder() {
+ this(createDefaultGroupInformation());
+ }
+
+ public ConsumerNetworkThreadTestBuilder(Optional<GroupInformation> groupInfo) {
+ super(groupInfo);
+ this.consumerNetworkThread = new ConsumerNetworkThread(
+ logContext,
+ time,
+ () -> applicationEventProcessor,
+ () -> networkClientDelegate,
+ () -> requestManagers
+ );
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(consumerNetworkThread, ConsumerNetworkThread.class.getSimpleName());
+ }
+ }
+
+ public static class ApplicationEventHandlerTestBuilder extends ConsumerTestBuilder {
+
+ public final ApplicationEventHandler applicationEventHandler;
+
+ public ApplicationEventHandlerTestBuilder() {
+ this(createDefaultGroupInformation());
+ }
+
+ public ApplicationEventHandlerTestBuilder(Optional<GroupInformation> groupInfo) {
+ super(groupInfo);
+ this.applicationEventHandler = spy(new ApplicationEventHandler(
+ logContext,
+ time,
+ applicationEventQueue,
+ () -> applicationEventProcessor,
+ () -> networkClientDelegate,
+ () -> requestManagers));
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(applicationEventHandler, ApplicationEventHandler.class.getSimpleName());
+ }
+ }
+
+ public static class PrototypeAsyncConsumerTestBuilder extends ApplicationEventHandlerTestBuilder {
+
+ final PrototypeAsyncConsumer<String, String> consumer;
+
+ public PrototypeAsyncConsumerTestBuilder(Optional<GroupInformation> groupInfo) {
+ super(groupInfo);
+ String clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG);
+ List<ConsumerPartitionAssignor> assignors = ConsumerPartitionAssignor.getAssignorInstances(
+ config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG),
+ config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId))
+ );
+ Deserializers<String, String> deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer());
+ FetchCollector<String, String> fetchCollector = new FetchCollector<>(logContext,
+ metadata,
+ subscriptions,
+ fetchConfig,
+ deserializers,
+ metricsManager,
+ time);
+ this.consumer = spy(new PrototypeAsyncConsumer<>(
+ logContext,
+ clientId,
+ deserializers,
+ new FetchBuffer(logContext),
+ fetchCollector,
+ new ConsumerInterceptors<>(Collections.emptyList()),
+ time,
+ applicationEventHandler,
+ backgroundEventQueue,
+ metrics,
+ subscriptions,
+ metadata,
+ retryBackoffMs,
+ 60000,
+ assignors,
+ groupInfo.map(groupInformation -> groupInformation.groupState.groupId).orElse(null)));
+ }
+
+ @Override
+ public void close() {
+ consumer.close();
+ }
+ }
+
+ public static class GroupInformation {
+
+ final GroupState groupState;
+ final int heartbeatIntervalMs;
+ final double heartbeatJitterMs;
+
+ public GroupInformation(GroupState groupState) {
+ this(groupState, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_JITTER_MS);
+ }
+
+ public GroupInformation(GroupState groupState, int heartbeatIntervalMs, double heartbeatJitterMs) {
+ this.groupState = groupState;
+ this.heartbeatIntervalMs = heartbeatIntervalMs;
+ this.heartbeatJitterMs = heartbeatJitterMs;
+ }
+ }
+
+ static Optional<GroupInformation> createDefaultGroupInformation() {
+ return Optional.of(new GroupInformation(new GroupState(DEFAULT_GROUP_ID, Optional.empty())));
+ }
+}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java
index 4018f48fba8..23ee97daf45 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java
@@ -17,6 +17,8 @@
package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.errors.GroupAuthorizationException;
import org.apache.kafka.common.errors.TimeoutException;
@@ -46,14 +48,14 @@ public class CoordinatorRequestManagerTest {
private static final int RETRY_BACKOFF_MS = 500;
private static final String GROUP_ID = "group-1";
private MockTime time;
- private ErrorEventHandler errorEventHandler;
+ private BackgroundEventHandler backgroundEventHandler;
private Node node;
@BeforeEach
public void setup() {
this.time = new MockTime(0);
this.node = new Node(1, "localhost", 9092);
- this.errorEventHandler = mock(ErrorEventHandler.class);
+ this.backgroundEventHandler = mock(BackgroundEventHandler.class);
}
@Test
@@ -97,7 +99,7 @@ public class CoordinatorRequestManagerTest {
public void testBackoffAfterRetriableFailure() {
CoordinatorRequestManager coordinatorManager = setupCoordinatorManager(GROUP_ID);
expectFindCoordinatorRequest(coordinatorManager, Errors.COORDINATOR_LOAD_IN_PROGRESS);
- verifyNoInteractions(errorEventHandler);
+ verifyNoInteractions(backgroundEventHandler);
time.sleep(RETRY_BACKOFF_MS - 1);
assertEquals(Collections.emptyList(), coordinatorManager.poll(time.milliseconds()).unsentRequests);
@@ -111,10 +113,15 @@ public class CoordinatorRequestManagerTest {
CoordinatorRequestManager coordinatorManager = setupCoordinatorManager(GROUP_ID);
expectFindCoordinatorRequest(coordinatorManager, Errors.GROUP_AUTHORIZATION_FAILED);
- verify(errorEventHandler).handle(argThat(exception -> {
- if (!(exception instanceof GroupAuthorizationException)) {
+ verify(backgroundEventHandler).add(argThat(backgroundEvent -> {
+ if (!(backgroundEvent instanceof ErrorBackgroundEvent))
return false;
- }
+
+ RuntimeException exception = ((ErrorBackgroundEvent) backgroundEvent).error();
+
+ if (!(exception instanceof GroupAuthorizationException))
+ return false;
+
GroupAuthorizationException groupAuthException = (GroupAuthorizationException) exception;
return groupAuthException.groupId().equals(GROUP_ID);
}));
@@ -185,7 +192,7 @@ public class CoordinatorRequestManagerTest {
new LogContext(),
RETRY_BACKOFF_MS,
RETRY_BACKOFF_MS,
- this.errorEventHandler,
+ this.backgroundEventHandler,
groupId
);
}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java
deleted file mode 100644
index a7d8ac0a61a..00000000000
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals;
-
-import org.apache.kafka.clients.GroupRebalanceConfig;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.LogTruncationException;
-import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor;
-import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
-import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent;
-import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.errors.TopicAuthorizationException;
-import org.apache.kafka.common.message.FindCoordinatorRequestData;
-import org.apache.kafka.common.requests.FindCoordinatorRequest;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.utils.LogContext;
-import org.apache.kafka.common.utils.MockTime;
-import org.apache.kafka.common.utils.Time;
-import org.apache.kafka.test.TestUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
-import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG;
-import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-@SuppressWarnings("ClassDataAbstractionCoupling")
-public class DefaultBackgroundThreadTest {
- private static final long RETRY_BACKOFF_MS = 100;
- private final Properties properties = new Properties();
- private MockTime time;
- private ConsumerMetadata metadata;
- private NetworkClientDelegate networkClient;
- private BlockingQueue<BackgroundEvent> backgroundEventsQueue;
- private BlockingQueue<ApplicationEvent> applicationEventsQueue;
- private ApplicationEventProcessor applicationEventProcessor;
- private CoordinatorRequestManager coordinatorManager;
- private OffsetsRequestManager offsetsRequestManager;
- private ErrorEventHandler errorEventHandler;
- private final int requestTimeoutMs = 500;
- private GroupState groupState;
- private CommitRequestManager commitManager;
- private TopicMetadataRequestManager topicMetadataRequestManager;
- private HeartbeatRequestManager heartbeatRequestManager;
-
- @BeforeEach
- @SuppressWarnings("unchecked")
- public void setup() {
- this.time = new MockTime(0);
- this.metadata = mock(ConsumerMetadata.class);
- this.networkClient = mock(NetworkClientDelegate.class);
- this.applicationEventsQueue = (BlockingQueue<ApplicationEvent>) mock(BlockingQueue.class);
- this.backgroundEventsQueue = (BlockingQueue<BackgroundEvent>) mock(BlockingQueue.class);
- this.applicationEventProcessor = mock(ApplicationEventProcessor.class);
- this.coordinatorManager = mock(CoordinatorRequestManager.class);
- this.offsetsRequestManager = mock(OffsetsRequestManager.class);
- this.heartbeatRequestManager = mock(HeartbeatRequestManager.class);
- this.errorEventHandler = mock(ErrorEventHandler.class);
- GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig(
- 100,
- 100,
- 100,
- "group_id",
- Optional.empty(),
- 100,
- 1000,
- true);
- this.groupState = new GroupState(rebalanceConfig);
- this.commitManager = mock(CommitRequestManager.class);
- this.topicMetadataRequestManager = mock(TopicMetadataRequestManager.class);
- }
-
- @Test
- public void testStartupAndTearDown() throws InterruptedException {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- backgroundThread.start();
- TestUtils.waitForCondition(backgroundThread::isRunning, "Failed awaiting for the background thread to be running");
- backgroundThread.close();
- assertFalse(backgroundThread.isRunning());
- }
-
- @Test
- public void testApplicationEvent() {
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- ApplicationEvent e = new NoopApplicationEvent("noop event");
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(applicationEventProcessor, times(1)).process(e);
- backgroundThread.close();
- }
-
- @Test
- public void testMetadataUpdateEvent() {
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- this.applicationEventProcessor = new ApplicationEventProcessor(
- this.backgroundEventsQueue,
- mockRequestManagers(),
- metadata);
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- ApplicationEvent e = new NewTopicsMetadataUpdateRequestEvent();
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(metadata).requestUpdateForNewTopics();
- backgroundThread.close();
- }
-
- @Test
- public void testCommitEvent() {
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- ApplicationEvent e = new CommitApplicationEvent(new HashMap<>());
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(CommitApplicationEvent.class));
- backgroundThread.close();
- }
-
-
- @Test
- public void testListOffsetsEventIsProcessed() {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- Map<TopicPartition, Long> timestamps = Collections.singletonMap(new TopicPartition("topic1", 1), 5L);
- ApplicationEvent e = new ListOffsetsApplicationEvent(timestamps, true);
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(ListOffsetsApplicationEvent.class));
- assertTrue(applicationEventsQueue.isEmpty());
- backgroundThread.close();
- }
-
- @Test
- public void testResetPositionsEventIsProcessed() {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- ResetPositionsApplicationEvent e = new ResetPositionsApplicationEvent();
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class));
- assertTrue(applicationEventsQueue.isEmpty());
- backgroundThread.close();
- }
-
- @Test
- public void testResetPositionsProcessFailure() {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- applicationEventProcessor = spy(new ApplicationEventProcessor(
- this.backgroundEventsQueue,
- mockRequestManagers(),
- metadata));
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
-
- TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed");
- doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded();
-
- ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent();
- this.applicationEventsQueue.add(event);
- assertThrows(TopicAuthorizationException.class, backgroundThread::runOnce);
-
- verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class));
- backgroundThread.close();
- }
-
- @Test
- public void testValidatePositionsEventIsProcessed() {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- ValidatePositionsApplicationEvent e = new ValidatePositionsApplicationEvent();
- this.applicationEventsQueue.add(e);
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class));
- assertTrue(applicationEventsQueue.isEmpty());
- backgroundThread.close();
- }
-
- @Test
- public void testValidatePositionsProcessFailure() {
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- applicationEventProcessor = spy(new ApplicationEventProcessor(
- this.backgroundEventsQueue,
- mockRequestManagers(),
- metadata));
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
-
- LogTruncationException logTruncationException = new LogTruncationException(Collections.emptyMap(), Collections.emptyMap());
- doThrow(logTruncationException).when(offsetsRequestManager).validatePositionsIfNeeded();
-
- ValidatePositionsApplicationEvent event = new ValidatePositionsApplicationEvent();
- this.applicationEventsQueue.add(event);
- assertThrows(LogTruncationException.class, backgroundThread::runOnce);
-
- verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class));
- backgroundThread.close();
- }
-
- @Test
- public void testAssignmentChangeEvent() {
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- this.backgroundEventsQueue = new LinkedBlockingQueue<>();
- this.applicationEventProcessor = spy(new ApplicationEventProcessor(
- this.backgroundEventsQueue,
- mockRequestManagers(),
- metadata));
-
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- HashMap<TopicPartition, OffsetAndMetadata> offset = mockTopicPartitionOffset();
-
- final long currentTimeMs = time.milliseconds();
- ApplicationEvent e = new AssignmentChangeApplicationEvent(offset, currentTimeMs);
- this.applicationEventsQueue.add(e);
-
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
-
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(AssignmentChangeApplicationEvent.class));
- verify(networkClient, times(1)).poll(anyLong(), anyLong());
- verify(commitManager, times(1)).updateAutoCommitTimer(currentTimeMs);
- verify(commitManager, times(1)).maybeAutoCommit(offset);
-
- backgroundThread.close();
- }
-
- @Test
- void testFindCoordinator() {
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- backgroundThread.runOnce();
- Mockito.verify(coordinatorManager, times(1)).poll(anyLong());
- Mockito.verify(networkClient, times(1)).poll(anyLong(), anyLong());
- backgroundThread.close();
- }
-
- @Test
- void testFetchTopicMetadata() {
- this.applicationEventsQueue = new LinkedBlockingQueue<>();
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult());
- when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult());
- when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- when(heartbeatRequestManager.poll(anyLong())).thenReturn(emptyPollResults());
- this.applicationEventsQueue.add(new TopicMetadataApplicationEvent("topic"));
- backgroundThread.runOnce();
- verify(applicationEventProcessor).process(any(TopicMetadataApplicationEvent.class));
- backgroundThread.close();
- }
-
- @Test
- void testPollResultTimer() {
- DefaultBackgroundThread backgroundThread = mockBackgroundThread();
- // purposely setting a non MAX time to ensure it is returning Long.MAX_VALUE upon success
- NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult(
- 10,
- Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs)));
- assertEquals(10, backgroundThread.handlePollResult(success));
-
- NetworkClientDelegate.PollResult failure = new NetworkClientDelegate.PollResult(
- 10,
- new ArrayList<>());
- assertEquals(10, backgroundThread.handlePollResult(failure));
- }
-
- private HashMap<TopicPartition, OffsetAndMetadata> mockTopicPartitionOffset() {
- final TopicPartition t0 = new TopicPartition("t0", 2);
- final TopicPartition t1 = new TopicPartition("t0", 3);
- HashMap<TopicPartition, OffsetAndMetadata> topicPartitionOffsets = new HashMap<>();
- topicPartitionOffsets.put(t0, new OffsetAndMetadata(10L));
- topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L));
- return topicPartitionOffsets;
- }
-
- private RequestManagers mockRequestManagers() {
- return new RequestManagers(
- offsetsRequestManager,
- topicMetadataRequestManager,
- Optional.of(coordinatorManager),
- Optional.of(commitManager),
- Optional.of(heartbeatRequestManager));
- }
-
- private static NetworkClientDelegate.UnsentRequest findCoordinatorUnsentRequest(
- final Time time,
- final long timeout
- ) {
- NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest(
- new FindCoordinatorRequest.Builder(
- new FindCoordinatorRequestData()
- .setKeyType(FindCoordinatorRequest.CoordinatorType.TRANSACTION.id())
- .setKey("foobar")),
- Optional.empty());
- req.setTimer(time, timeout);
- return req;
- }
-
- private DefaultBackgroundThread mockBackgroundThread() {
- properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
- properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
- properties.put(RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS);
-
- return new DefaultBackgroundThread(
- this.time,
- new ConsumerConfig(properties),
- new LogContext(),
- applicationEventsQueue,
- backgroundEventsQueue,
- this.errorEventHandler,
- applicationEventProcessor,
- this.metadata,
- this.networkClient,
- this.groupState,
- this.coordinatorManager,
- this.commitManager,
- this.offsetsRequestManager,
- this.topicMetadataRequestManager,
- this.heartbeatRequestManager);
- }
-
- private NetworkClientDelegate.PollResult mockPollCoordinatorResult() {
- return new NetworkClientDelegate.PollResult(
- RETRY_BACKOFF_MS,
- Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs)));
- }
-
- private NetworkClientDelegate.PollResult mockPollCommitResult() {
- return new NetworkClientDelegate.PollResult(
- RETRY_BACKOFF_MS,
- Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs)));
- }
-
- private NetworkClientDelegate.PollResult emptyPollResults() {
- return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList());
- }
-}
\ No newline at end of file
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java
deleted file mode 100644
index 7e4de791da6..00000000000
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.consumer.internals;
-
-import org.apache.kafka.clients.GroupRebalanceConfig;
-import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent;
-import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent;
-import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
-import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG;
-import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-public class DefaultEventHandlerTest {
- private int sessionTimeoutMs = 1000;
- private int rebalanceTimeoutMs = 1000;
- private int heartbeatIntervalMs = 1000;
- private String groupId = "g-1";
- private Optional<String> groupInstanceId = Optional.of("g-1");
- private long retryBackoffMs = 1000;
- private final Properties properties = new Properties();
- private GroupRebalanceConfig rebalanceConfig;
-
- @BeforeEach
- public void setup() {
- properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
- properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
- properties.put(RETRY_BACKOFF_MS_CONFIG, "100");
-
- this.rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs,
- rebalanceTimeoutMs,
- heartbeatIntervalMs,
- groupId,
- groupInstanceId,
- retryBackoffMs,
- retryBackoffMs,
- true);
- }
-
- @Test
- public void testBasicHandlerOps() {
- final DefaultBackgroundThread bt = mock(DefaultBackgroundThread.class);
- final BlockingQueue<ApplicationEvent> aq = new LinkedBlockingQueue<>();
- final BlockingQueue<BackgroundEvent> bq = new LinkedBlockingQueue<>();
- final DefaultEventHandler handler = new DefaultEventHandler(bt, aq, bq);
- assertTrue(handler.isEmpty());
- assertFalse(handler.poll().isPresent());
- handler.add(new NoopApplicationEvent("test"));
- assertEquals(1, aq.size());
- handler.close();
- verify(bt, times(1)).close();
- }
-}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java
index 1f157b84189..0ca4a18a48a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java
@@ -51,7 +51,6 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Stream;
-import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics;
import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState;
@@ -79,10 +78,11 @@ public class FetchCollectorTest {
private LogContext logContext;
private SubscriptionState subscriptions;
- private FetchConfig<String, String> fetchConfig;
+ private FetchConfig fetchConfig;
private FetchMetricsManager metricsManager;
private ConsumerMetadata metadata;
private FetchBuffer fetchBuffer;
+ private Deserializers<String, String> deserializers;
private FetchCollector<String, String> fetchCollector;
private CompletedFetchBuilder completedFetchBuilder;
@@ -193,6 +193,7 @@ public class FetchCollectorTest {
metadata,
subscriptions,
fetchConfig,
+ deserializers,
metricsManager,
time) {
@@ -427,10 +428,10 @@ public class FetchCollectorTest {
ConsumerConfig config = new ConsumerConfig(p);
- Deserializers<String, String> deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer());
+ deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer());
subscriptions = createSubscriptionState(config, logContext);
- fetchConfig = createFetchConfig(config, deserializers);
+ fetchConfig = new FetchConfig(config);
Metrics metrics = createMetrics(config, time);
metricsManager = createFetchMetricsManager(metrics);
@@ -448,6 +449,7 @@ public class FetchCollectorTest {
metadata,
subscriptions,
fetchConfig,
+ deserializers,
metricsManager,
time);
fetchBuffer = new FetchBuffer(logContext);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java
index 1f89822fdb2..f38a6294624 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java
@@ -19,14 +19,11 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.IsolationLevel;
import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.Test;
import java.util.Properties;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
public class FetchConfigTest {
/**
@@ -35,60 +32,27 @@ public class FetchConfigTest {
*/
@Test
public void testBasicFromConsumerConfig() {
- try (StringDeserializer keyDeserializer = new StringDeserializer(); StringDeserializer valueDeserializer = new StringDeserializer()) {
- newFetchConfigFromConsumerConfig(keyDeserializer, valueDeserializer);
- newFetchConfigFromValues(keyDeserializer, valueDeserializer);
- }
- }
-
- /**
- * Verify an exception is thrown if the key {@link Deserializer deserializer} provided to the
- * {@link FetchConfig} constructors is {@code null}.
- */
- @Test
- public void testPreventNullKeyDeserializer() {
- try (StringDeserializer valueDeserializer = new StringDeserializer()) {
- assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(null, valueDeserializer));
- assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(null, valueDeserializer));
- }
+ newFetchConfigFromConsumerConfig();
+ newFetchConfigFromValues();
}
- /**
- * Verify an exception is thrown if the value {@link Deserializer deserializer} provided to the
- * {@link FetchConfig} constructors is {@code null}.
- */
- @Test
- @SuppressWarnings("resources")
- public void testPreventNullValueDeserializer() {
- try (StringDeserializer keyDeserializer = new StringDeserializer()) {
- assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(keyDeserializer, null));
- assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(keyDeserializer, null));
- }
- }
-
- private void newFetchConfigFromConsumerConfig(Deserializer<String> keyDeserializer,
- Deserializer<String> valueDeserializer) {
+ private void newFetchConfigFromConsumerConfig() {
Properties p = new Properties();
p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
ConsumerConfig config = new ConsumerConfig(p);
- new FetchConfig<>(
- config,
- new Deserializers<>(keyDeserializer, valueDeserializer),
- IsolationLevel.READ_UNCOMMITTED);
+ new FetchConfig(config);
}
- private void newFetchConfigFromValues(Deserializer<String> keyDeserializer,
- Deserializer<String> valueDeserializer) {
- new FetchConfig<>(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES,
+ private void newFetchConfigFromValues() {
+ new FetchConfig(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES,
ConsumerConfig.DEFAULT_FETCH_MAX_BYTES,
ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS,
ConsumerConfig.DEFAULT_MAX_PARTITION_FETCH_BYTES,
ConsumerConfig.DEFAULT_MAX_POLL_RECORDS,
true,
ConsumerConfig.DEFAULT_CLIENT_RACK,
- new Deserializers<>(keyDeserializer, valueDeserializer),
IsolationLevel.READ_UNCOMMITTED);
}
}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java
similarity index 76%
copy from clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
copy to clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java
index 6336c11219e..c4270c3f4ca 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java
@@ -19,11 +19,12 @@ package org.apache.kafka.clients.consumer.internals;
import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.ClientRequest;
import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.FetchSessionHandler;
+import org.apache.kafka.clients.KafkaClient;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.NetworkClient;
import org.apache.kafka.clients.NodeApiVersions;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.OffsetOutOfRangeException;
@@ -34,17 +35,18 @@ import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.MetricNameTemplate;
import org.apache.kafka.common.Node;
-import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.DisconnectException;
import org.apache.kafka.common.errors.RecordTooLargeException;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.errors.TopicAuthorizationException;
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
import org.apache.kafka.common.internals.ClusterResourceListeners;
-import org.apache.kafka.common.message.FetchResponseData;
import org.apache.kafka.common.message.ApiMessageType;
+import org.apache.kafka.common.message.FetchResponseData;
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData;
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset;
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult;
@@ -54,9 +56,6 @@ import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.network.NetworkReceive;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.requests.FetchMetadata;
-import org.apache.kafka.common.requests.FetchRequest.PartitionData;
-import org.apache.kafka.common.utils.BufferSupplier;
import org.apache.kafka.common.record.CompressionType;
import org.apache.kafka.common.record.ControlRecordType;
import org.apache.kafka.common.record.DefaultRecordBatch;
@@ -70,7 +69,9 @@ import org.apache.kafka.common.record.Records;
import org.apache.kafka.common.record.SimpleRecord;
import org.apache.kafka.common.record.TimestampType;
import org.apache.kafka.common.requests.ApiVersionsResponse;
+import org.apache.kafka.common.requests.FetchMetadata;
import org.apache.kafka.common.requests.FetchRequest;
+import org.apache.kafka.common.requests.FetchRequest.PartitionData;
import org.apache.kafka.common.requests.FetchResponse;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse;
@@ -79,9 +80,11 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.kafka.common.serialization.BytesDeserializer;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.utils.BufferSupplier;
import org.apache.kafka.common.utils.ByteBufferOutputStream;
import org.apache.kafka.common.utils.LogContext;
import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Timer;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.test.DelayedReceive;
@@ -90,10 +93,14 @@ import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
-import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
@@ -107,15 +114,12 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Properties;
import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
@@ -123,6 +127,8 @@ import static java.util.Collections.emptySet;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG;
+import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG;
import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID;
import static org.apache.kafka.common.utils.Utils.mkSet;
import static org.apache.kafka.test.TestUtils.assertOptional;
@@ -140,7 +146,8 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-public class FetcherTest {
+public class FetchRequestManagerTest {
+
private static final double EPSILON = 0.0001;
private ConsumerRebalanceListener listener = new NoOpConsumerRebalanceListener();
@@ -164,7 +171,7 @@ public class FetcherTest {
private TopicIdPartition tidp3 = new TopicIdPartition(topicId, tp3);
private int validLeaderEpoch = 0;
private MetadataResponse initialUpdateResponse =
- RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, 4), topicIds);
+ RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, 4), topicIds);
private int minBytes = 1;
private int maxBytes = Integer.MAX_VALUE;
@@ -180,15 +187,14 @@ public class FetcherTest {
private MockClient client;
private Metrics metrics;
private ApiVersions apiVersions = new ApiVersions();
- private ConsumerNetworkClient consumerClient;
- private Fetcher<?, ?> fetcher;
+ private TestableFetchRequestManager<?, ?> fetcher;
+ private TestableNetworkClientDelegate networkClientDelegate;
private OffsetFetcher offsetFetcher;
private MemoryRecords records;
private MemoryRecords nextRecords;
private MemoryRecords emptyRecords;
private MemoryRecords partialRecords;
- private ExecutorService executorService;
@BeforeEach
public void setup() {
@@ -205,30 +211,26 @@ public class FetcherTest {
// A dummy metadata update to ensure valid leader epoch.
metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWithIds("dummy", 1,
- Collections.emptyMap(), singletonMap(topicName, 4),
- tp -> validLeaderEpoch, topicIds), false, 0L);
+ Collections.emptyMap(), singletonMap(topicName, 4),
+ tp -> validLeaderEpoch, topicIds), false, 0L);
}
- private void assignFromUserNoId(Set<TopicPartition> partitions) {
- subscriptions.assignFromUser(partitions);
- client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap()));
+ private void assignFromUser(TopicPartition partition) {
+ subscriptions.assignFromUser(singleton(partition));
+ client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap(partition.topic(), 1), Collections.emptyMap()));
// A dummy metadata update to ensure valid leader epoch.
metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1,
- Collections.emptyMap(), singletonMap("noId", 1),
- tp -> validLeaderEpoch, topicIds), false, 0L);
+ Collections.emptyMap(), singletonMap(partition.topic(), 1),
+ tp -> validLeaderEpoch, topicIds), false, 0L);
}
@AfterEach
public void teardown() throws Exception {
if (metrics != null)
- this.metrics.close();
+ metrics.close();
if (fetcher != null)
- this.fetcher.close();
- if (executorService != null) {
- executorService.shutdownNow();
- assertTrue(executorService.awaitTermination(5, TimeUnit.SECONDS));
- }
+ fetcher.close();
}
private int sendFetches() {
@@ -247,11 +249,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp0);
@@ -274,9 +276,9 @@ public class FetcherTest {
assertEquals(1, sendFetches());
subscriptions.markPendingRevocation(singleton(tp0));
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
- assertNull(fetchedRecords().get(tp0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
+ assertNull(fetchRecords().get(tp0));
}
@Test
@@ -287,7 +289,7 @@ public class FetcherTest {
fetcher.close();
fetcher.close();
- verify(fetcher, times(1)).maybeCloseFetchSessions(any(Timer.class));
+ verify(fetcher, times(1)).closeInternal(any(Timer.class));
}
@Test
@@ -301,21 +303,25 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0);
+ final FetchResponse fetchResponse = fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0);
client.prepareResponse(fetchResponse);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- assertEquals(0, consumerClient.pendingRequestCount());
+ assertEquals(0, networkClientDelegate.pendingRequestCount());
- final ArgumentCaptor<FetchRequest.Builder> argument = ArgumentCaptor.forClass(FetchRequest.Builder.class);
+ final ArgumentCaptor<NetworkClientDelegate.UnsentRequest> argument = ArgumentCaptor.forClass(NetworkClientDelegate.UnsentRequest.class);
- // send request to close the fetcher
- this.fetcher.close(time.timer(Duration.ofSeconds(10)));
+ Timer timer = time.timer(Duration.ofSeconds(10));
+ // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling
+ // the close() method with a Timer will NOT send out the close session requests on close. The network
+ // I/O logic is handled inside ConsumerNetworkThread.runAtClose, so we need to run that logic here.
+ ConsumerNetworkThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer);
- // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal
- // fetch earlier and another for the finish fetch here.
- verify(consumerClient, times(2)).send(any(Node.class), argument.capture());
- FetchRequest.Builder builder = argument.getValue();
+ // validate that closing the fetcher has sent a request with final epoch. 2 requests are sent, one for the
+ // normal fetch earlier and another for the finish fetch here.
+ verify(networkClientDelegate, times(2)).doSend(argument.capture(), any(Long.class));
+ NetworkClientDelegate.UnsentRequest lastUnsentRequest = argument.getValue();
+ FetchRequest.Builder builder = (FetchRequest.Builder) lastUnsentRequest.requestBuilder();
// session Id is the same
assertEquals(fetchResponse.sessionId(), builder.metadata().sessionId());
// contains final epoch
@@ -332,18 +338,18 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position
// mark partition unfetchable
subscriptions.markPendingRevocation(singleton(tp0));
assertEquals(0, sendFetches());
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertFalse(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
assertEquals(4L, subscriptions.position(tp0).offset);
}
@@ -353,21 +359,21 @@ public class FetcherTest {
buildFetcher();
TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0));
- assignFromUserNoId(singleton(noId.topicPartition()));
+ assignFromUser(noId.topicPartition());
subscriptions.seek(noId.topicPartition(), 0);
- // Fetch should use request version 12
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
+ // Fetch should use request version 12
client.prepareResponse(
- fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)),
- fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0)
+ fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)),
+ fullFetchResponse(noId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(noId.topicPartition()));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(noId.topicPartition());
@@ -388,18 +394,18 @@ public class FetcherTest {
assignFromUser(singleton(tp.topicPartition()));
subscriptions.seek(tp.topicPartition(), 0);
- // Fetch should use latest version
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
+ // Fetch should use latest version
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)),
- fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0)
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)),
+ fullFetchResponse(tp, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp.topicPartition()));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp.topicPartition());
@@ -424,27 +430,27 @@ public class FetcherTest {
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch));
subscriptions.seek(foo.topicPartition(), 0);
- // Fetch should use latest version.
assertEquals(1, sendFetches());
+ // Fetch should use latest version.
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
- singletonMap(foo, new PartitionData(
- foo.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
+ singletonMap(foo, new PartitionData(
+ foo.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ emptyList()
),
- emptyList()
- ),
- fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, foo, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
- // Assign bar and unassign foo.
+ // Assign bar and un-assign foo.
subscriptions.assignFromUser(singleton(bar.topicPartition()));
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch));
subscriptions.seek(bar.topicPartition(), 0);
@@ -454,21 +460,21 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
- singletonMap(bar, new PartitionData(
- bar.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
+ singletonMap(bar, new PartitionData(
+ bar.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ singletonList(foo)
),
- singletonList(foo)
- ),
- fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, bar, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
@Test
@@ -487,21 +493,21 @@ public class FetcherTest {
assertEquals(1, sendFetches());
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
- singletonMap(fooWithOldTopicId, new PartitionData(
- fooWithOldTopicId.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
+ singletonMap(fooWithOldTopicId, new PartitionData(
+ fooWithOldTopicId.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ emptyList()
),
- emptyList()
- ),
- fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithOldTopicId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Replace foo with old topic id with foo with new topic id.
subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition()));
@@ -514,21 +520,21 @@ public class FetcherTest {
// foo with old topic id should be removed from the session.
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
- singletonMap(fooWithNewTopicId, new PartitionData(
- fooWithNewTopicId.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
+ singletonMap(fooWithNewTopicId, new PartitionData(
+ fooWithNewTopicId.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ singletonList(fooWithOldTopicId)
),
- singletonList(fooWithOldTopicId)
- ),
- fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithNewTopicId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
@Test
@@ -536,7 +542,6 @@ public class FetcherTest {
buildFetcher();
TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0));
- TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
// Assign foo without a topic id.
subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition()));
@@ -547,23 +552,24 @@ public class FetcherTest {
assertEquals(1, sendFetches());
client.prepareResponse(
- fetchRequestMatcher((short) 12,
- singletonMap(fooWithoutId, new PartitionData(
- fooWithoutId.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher((short) 12,
+ singletonMap(fooWithoutId, new PartitionData(
+ fooWithoutId.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ emptyList()
),
- emptyList()
- ),
- fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Upgrade.
+ TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
subscriptions.assignFromUser(singleton(fooWithId.topicPartition()));
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch));
subscriptions.seek(fooWithId.topicPartition(), 0);
@@ -574,21 +580,21 @@ public class FetcherTest {
// foo with old topic id should be removed from the session.
client.prepareResponse(
- fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
- singletonMap(fooWithId, new PartitionData(
- fooWithId.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
+ singletonMap(fooWithId, new PartitionData(
+ fooWithId.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ emptyList()
),
- emptyList()
- ),
- fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Downgrade.
subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition()));
@@ -601,46 +607,46 @@ public class FetcherTest {
// foo with old topic id should be removed from the session.
client.prepareResponse(
- fetchRequestMatcher((short) 12,
- singletonMap(fooWithoutId, new PartitionData(
- fooWithoutId.topicId(),
- 0,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- Optional.of(validLeaderEpoch))
+ fetchRequestMatcher((short) 12,
+ singletonMap(fooWithoutId, new PartitionData(
+ fooWithoutId.topicId(),
+ 0,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ Optional.of(validLeaderEpoch))
+ ),
+ emptyList()
),
- emptyList()
- ),
- fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0)
);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
private MockClient.RequestMatcher fetchRequestMatcher(
- short expectedVersion,
- TopicIdPartition tp,
- long expectedFetchOffset,
- Optional<Integer> expectedCurrentLeaderEpoch
+ short expectedVersion,
+ TopicIdPartition tp,
+ long expectedFetchOffset,
+ Optional<Integer> expectedCurrentLeaderEpoch
) {
return fetchRequestMatcher(
- expectedVersion,
- singletonMap(tp, new PartitionData(
- tp.topicId(),
- expectedFetchOffset,
- FetchRequest.INVALID_LOG_START_OFFSET,
- fetchSize,
- expectedCurrentLeaderEpoch
- )),
- emptyList()
+ expectedVersion,
+ singletonMap(tp, new PartitionData(
+ tp.topicId(),
+ expectedFetchOffset,
+ FetchRequest.INVALID_LOG_START_OFFSET,
+ fetchSize,
+ expectedCurrentLeaderEpoch
+ )),
+ emptyList()
);
}
private MockClient.RequestMatcher fetchRequestMatcher(
- short expectedVersion,
- Map<TopicIdPartition, PartitionData> fetch,
- List<TopicIdPartition> forgotten
+ short expectedVersion,
+ Map<TopicIdPartition, PartitionData> fetch,
+ List<TopicIdPartition> forgotten
) {
return body -> {
if (body instanceof FetchRequest) {
@@ -681,10 +687,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
assertEquals(2, partitionRecords.get(tp0).size());
@@ -732,10 +738,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
assertEquals(6, partitionRecords.get(tp0).size());
@@ -756,8 +762,8 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
Set<TopicPartition> newAssignedTopicPartitions = new HashSet<>();
newAssignedTopicPartitions.add(tp1);
@@ -809,10 +815,10 @@ public class FetcherTest {
buffer.flip();
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp0);
@@ -833,11 +839,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertFalse(partitionRecords.containsKey(tp0));
}
@@ -872,19 +878,16 @@ public class FetcherTest {
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 1);
- client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- // The fetcher should block on Deserialization error
+ networkClientDelegate.poll(time.timer(0));
+
for (int i = 0; i < 2; i++) {
- try {
- fetcher.collectFetch();
- fail("fetchedRecords should have raised");
- } catch (SerializationException e) {
- // the position should not advance since no data has been returned
- assertEquals(1, subscriptions.position(tp0).offset);
- }
+ // The fetcher should throw a Deserialization error
+ assertThrows(SerializationException.class, this::collectFetch);
+ // the position should not advance since no data has been returned
+ assertEquals(1, subscriptions.position(tp0).offset);
}
}
@@ -937,10 +940,10 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- // the first fetchedRecords() should return the first valid message
- assertEquals(1, fetchedRecords().get(tp0).size());
+ // the first fetchRecords() should return the first valid message
+ assertEquals(1, fetchRecords().get(tp0).size());
assertEquals(1, subscriptions.position(tp0).offset);
ensureBlockOnRecord(1L);
@@ -951,20 +954,16 @@ public class FetcherTest {
seekAndConsumeRecord(buffer, 4L);
fail("Should have thrown exception when fail to retrieve a record from iterator.");
} catch (KafkaException ke) {
- // let it go
+ // let it go
}
ensureBlockOnRecord(4L);
}
private void ensureBlockOnRecord(long blockedOffset) {
- // the fetchedRecords() should always throw exception due to the invalid message at the starting offset.
for (int i = 0; i < 2; i++) {
- try {
- fetcher.collectFetch();
- fail("fetchedRecords should have raised KafkaException");
- } catch (KafkaException e) {
- assertEquals(blockedOffset, subscriptions.position(tp0).offset);
- }
+ // the fetchRecords() should always throw exception due to the invalid message at the starting offset.
+ assertThrows(KafkaException.class, this::fetchRecords);
+ assertEquals(blockedOffset, subscriptions.position(tp0).offset);
}
}
@@ -972,12 +971,12 @@ public class FetcherTest {
// Seek to skip the bad record and fetch again.
subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0)));
// Should not throw exception after the seek.
- fetcher.collectFetch();
+ collectFetch();
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchRecords();
List<ConsumerRecord<byte[], byte[]>> records = recordsByPartition.get(tp0);
assertEquals(1, records.size());
assertEquals(toOffset, records.get(0).offset());
@@ -992,10 +991,10 @@ public class FetcherTest {
ByteBufferOutputStream out = new ByteBufferOutputStream(buffer);
MemoryRecordsBuilder builder = new MemoryRecordsBuilder(out,
- DefaultRecordBatch.CURRENT_MAGIC_VALUE,
- CompressionType.NONE,
- TimestampType.CREATE_TIME,
- 0L, 10L, 0L, (short) 0, 0, false, false, 0, 1024);
+ DefaultRecordBatch.CURRENT_MAGIC_VALUE,
+ CompressionType.NONE,
+ TimestampType.CREATE_TIME,
+ 0L, 10L, 0L, (short) 0, 0, false, false, 0, 1024);
builder.append(10L, "key".getBytes(), "value".getBytes());
builder.close();
buffer.flip();
@@ -1011,16 +1010,12 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- // the fetchedRecords() should always throw exception due to the bad batch.
for (int i = 0; i < 2; i++) {
- try {
- fetcher.collectFetch();
- fail("fetchedRecords should have raised KafkaException");
- } catch (KafkaException e) {
- assertEquals(0, subscriptions.position(tp0).offset);
- }
+ // the fetchRecords() should always throw exception due to the bad batch.
+ assertThrows(KafkaException.class, this::collectFetch);
+ assertEquals(0, subscriptions.position(tp0).offset);
}
}
@@ -1043,14 +1038,11 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
- try {
- fetcher.collectFetch();
- fail("fetchedRecords should have raised");
- } catch (KafkaException e) {
- // the position should not advance since no data has been returned
- assertEquals(0, subscriptions.position(tp0).offset);
- }
+ networkClientDelegate.poll(time.timer(0));
+
+ assertThrows(KafkaException.class, this::collectFetch);
+ // the position should not advance since no data has been returned
+ assertEquals(0, subscriptions.position(tp0).offset);
}
@Test
@@ -1078,8 +1070,8 @@ public class FetcherTest {
client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, memoryRecords, Errors.NONE, 100L, 0));
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchRecords();
records = recordsByPartition.get(tp0);
assertEquals(3, records.size());
@@ -1102,38 +1094,37 @@ public class FetcherTest {
public void testFetchMaxPollRecords() {
buildFetcher(2);
- List<ConsumerRecord<byte[], byte[]>> records;
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 1);
- client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, this.nextRecords, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0));
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
- records = recordsByPartition.get(tp0);
- assertEquals(2, records.size());
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchRecords();
+ List<ConsumerRecord<byte[], byte[]>> recordsToTest = recordsByPartition.get(tp0);
+ assertEquals(2, recordsToTest.size());
assertEquals(3L, subscriptions.position(tp0).offset);
- assertEquals(1, records.get(0).offset());
- assertEquals(2, records.get(1).offset());
+ assertEquals(1, recordsToTest.get(0).offset());
+ assertEquals(2, recordsToTest.get(1).offset());
assertEquals(0, sendFetches());
- consumerClient.poll(time.timer(0));
- recordsByPartition = fetchedRecords();
- records = recordsByPartition.get(tp0);
- assertEquals(1, records.size());
+ networkClientDelegate.poll(time.timer(0));
+ recordsByPartition = fetchRecords();
+ recordsToTest = recordsByPartition.get(tp0);
+ assertEquals(1, recordsToTest.size());
assertEquals(4L, subscriptions.position(tp0).offset);
- assertEquals(3, records.get(0).offset());
+ assertEquals(3, recordsToTest.get(0).offset());
assertTrue(sendFetches() > 0);
- consumerClient.poll(time.timer(0));
- recordsByPartition = fetchedRecords();
- records = recordsByPartition.get(tp0);
- assertEquals(2, records.size());
+ networkClientDelegate.poll(time.timer(0));
+ recordsByPartition = fetchRecords();
+ recordsToTest = recordsByPartition.get(tp0);
+ assertEquals(2, recordsToTest.size());
assertEquals(6L, subscriptions.position(tp0).offset);
- assertEquals(4, records.get(0).offset());
- assertEquals(5, records.get(1).offset());
+ assertEquals(4, recordsToTest.get(0).offset());
+ assertEquals(5, recordsToTest.get(1).offset());
}
/**
@@ -1145,35 +1136,34 @@ public class FetcherTest {
public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() {
buildFetcher(2);
- List<ConsumerRecord<byte[], byte[]>> records;
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 1);
// Returns 3 records while `max.poll.records` is configured to 2
- client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
- records = recordsByPartition.get(tp0);
- assertEquals(2, records.size());
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchRecords();
+ List<ConsumerRecord<byte[], byte[]>> recordsToTest = recordsByPartition.get(tp0);
+ assertEquals(2, recordsToTest.size());
assertEquals(3L, subscriptions.position(tp0).offset);
- assertEquals(1, records.get(0).offset());
- assertEquals(2, records.get(1).offset());
+ assertEquals(1, recordsToTest.get(0).offset());
+ assertEquals(2, recordsToTest.get(1).offset());
assignFromUser(singleton(tp1));
- client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0));
subscriptions.seek(tp1, 4);
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertNull(fetchedRecords.get(tp0));
- records = fetchedRecords.get(tp1);
- assertEquals(2, records.size());
+ recordsToTest = fetchedRecords.get(tp1);
+ assertEquals(2, recordsToTest.size());
assertEquals(6L, subscriptions.position(tp1).offset);
- assertEquals(4, records.get(0).offset());
- assertEquals(5, records.get(1).offset());
+ assertEquals(4, recordsToTest.get(0).offset());
+ assertEquals(5, recordsToTest.get(1).offset());
}
@Test
@@ -1196,8 +1186,8 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchRecords();
consumerRecords = recordsByPartition.get(tp0);
assertEquals(3, consumerRecords.size());
assertEquals(31L, subscriptions.position(tp0).offset); // this is the next fetching position
@@ -1218,14 +1208,9 @@ public class FetcherTest {
client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2));
makeFetchRequestWithIncompleteRecord();
- try {
- fetcher.collectFetch();
- fail("RecordTooLargeException should have been raised");
- } catch (RecordTooLargeException e) {
- assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: "));
- // the position should not advance since no data has been returned
- assertEquals(0, subscriptions.position(tp0).offset);
- }
+ assertThrows(RecordTooLargeException.class, this::collectFetch);
+ // the position should not advance since no data has been returned
+ assertEquals(0, subscriptions.position(tp0).offset);
} finally {
client.setNodeApiVersions(NodeApiVersions.create());
}
@@ -1242,8 +1227,8 @@ public class FetcherTest {
buildFetcher();
makeFetchRequestWithIncompleteRecord();
try {
- fetcher.collectFetch();
- fail("RecordTooLargeException should have been raised");
+ collectFetch();
+ fail("collectFetch should have thrown a KafkaException");
} catch (KafkaException e) {
assertTrue(e.getMessage().startsWith("Failed to make progress reading messages"));
// the position should not advance since no data has been returned
@@ -1257,9 +1242,9 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
MemoryRecords partialRecord = MemoryRecords.readableRecords(
- ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0}));
+ ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0}));
client.prepareResponse(fullFetchResponse(tidp0, partialRecord, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
}
@@ -1270,13 +1255,12 @@ public class FetcherTest {
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 0);
- // resize the limit of the buffer to pretend it is only fetch-size large
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
try {
- fetcher.collectFetch();
- fail("fetchedRecords should have thrown");
+ collectFetch();
+ fail("collectFetch should have thrown a TopicAuthorizationException");
} catch (TopicAuthorizationException e) {
assertEquals(singleton(topicName), e.unauthorizedTopics());
}
@@ -1291,7 +1275,7 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(
- 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
+ 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
assertEquals(1, sendFetches());
@@ -1299,11 +1283,11 @@ public class FetcherTest {
subscriptions.assignFromSubscribed(Collections.emptyList());
subscriptions.assignFromSubscribed(singleton(tp0));
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
// The active fetch should be ignored since its position is no longer valid
- assertTrue(fetchedRecords().isEmpty());
+ assertTrue(fetchRecords().isEmpty());
}
@Test
@@ -1315,17 +1299,17 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(
- 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
+ 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
assertEquals(1, sendFetches());
// Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions
subscriptions.assignFromSubscribed(singleton(tp0));
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
// The active fetch should NOT be ignored since the position for tp0 is still valid
assertEquals(1, fetchedRecords.size());
@@ -1342,9 +1326,9 @@ public class FetcherTest {
assertEquals(1, sendFetches());
subscriptions.pause(tp0);
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
- assertNull(fetchedRecords().get(tp0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
+ assertNull(fetchRecords().get(tp0));
}
@Test
@@ -1370,8 +1354,8 @@ public class FetcherTest {
subscriptions.pause(tp0);
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return any records or advance position when partition is paused");
@@ -1383,13 +1367,13 @@ public class FetcherTest {
assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches");
- consumerClient.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed");
assertNotNull(fetchedRecords.get(tp0));
assertEquals(3, fetchedRecords.get(tp0).size());
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position after previously paused partitions are fetched");
assertFalse(fetcher.hasCompletedFetches(), "Should no longer contain completed fetches");
}
@@ -1407,18 +1391,18 @@ public class FetcherTest {
// #1 seek, request, poll, response
subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0)));
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
// #2 seek, request, poll, response
subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1)));
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0));
subscriptions.pause(tp0);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- fetchedRecords = fetchedRecords();
+ fetchedRecords = fetchRecords();
assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions");
assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches");
assertNotNull(fetchedRecords.get(tp1));
@@ -1439,18 +1423,18 @@ public class FetcherTest {
// #1 seek, request, poll, response
subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0)));
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
// #2 seek, request, poll, response
subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1)));
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0));
subscriptions.pause(tp0);
subscriptions.pause(tp1);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position for all paused partitions");
assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches");
@@ -1461,7 +1445,7 @@ public class FetcherTest {
public void testPartialFetchWithPausedPartitions() {
// this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert
// that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is
- // paused, then returned successfully after its been resumed again later
+ // paused, then returned successfully after it has been resumed again later
buildFetcher(2);
Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords;
@@ -1470,18 +1454,16 @@ public class FetcherTest {
subscriptions.seek(tp0, 1);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
- fetchedRecords = fetchedRecords();
+ fetchedRecords = fetchRecords();
assertEquals(2, fetchedRecords.get(tp0).size(), "Should return 2 records from fetch with 3 records");
assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches");
subscriptions.pause(tp0);
- consumerClient.poll(time.timer(0));
-
- fetchedRecords = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position for paused partitions");
assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches");
@@ -1489,74 +1471,35 @@ public class FetcherTest {
subscriptions.resume(tp0);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- fetchedRecords = fetchedRecords();
+ fetchedRecords = fetchRecords();
assertEquals(1, fetchedRecords.get(tp0).size(), "Should return last remaining record");
assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches");
}
@Test
- public void testFetchDiscardedAfterPausedPartitionResumedAndSoughtToNewOffset() {
+ public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() {
buildFetcher();
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
subscriptions.pause(tp0);
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
subscriptions.seek(tp0, 3);
subscriptions.resume(tp0);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches");
Fetch<byte[], byte[]> fetch = collectFetch();
- assertEquals(emptyMap(), fetch.records(), "Should not return any records because we sought to a new offset");
+ assertEquals(emptyMap(), fetch.records(), "Should not return any records because we seeked to a new offset");
assertFalse(fetch.positionAdvanced());
assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches");
}
- @Test
- public void testFetchNotLeaderOrFollower() {
- buildFetcher();
- assignFromUser(singleton(tp0));
- subscriptions.seek(tp0, 0);
-
- assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0));
- consumerClient.poll(time.timer(0));
- assertEmptyFetch("Should not return records or advance position on fetch error");
- assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
- }
-
- @Test
- public void testFetchUnknownTopicOrPartition() {
- buildFetcher();
- assignFromUser(singleton(tp0));
- subscriptions.seek(tp0, 0);
-
- assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0));
- consumerClient.poll(time.timer(0));
- assertEmptyFetch("Should not return records or advance position on fetch error");
- assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
- }
-
- @Test
- public void testFetchUnknownTopicId() {
- buildFetcher();
- assignFromUser(singleton(tp0));
- subscriptions.seek(tp0, 0);
-
- assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_ID, -1L, 0));
- consumerClient.poll(time.timer(0));
- assertEmptyFetch("Should not return records or advance position on fetch error");
- assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
- }
-
@Test
public void testFetchSessionIdError() {
buildFetcher();
@@ -1565,50 +1508,56 @@ public class FetcherTest {
assertEquals(1, sendFetches());
client.prepareResponse(fetchResponseWithTopLevelError(tidp0, Errors.FETCH_SESSION_TOPIC_ID_ERROR, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position on fetch error");
assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
}
- @Test
- public void testFetchInconsistentTopicId() {
+ @ParameterizedTest
+ @MethodSource("handleFetchResponseErrorSupplier")
+ public void testHandleFetchResponseError(Errors error,
+ long highWatermark,
+ boolean hasTopLevelError,
+ boolean shouldRequestMetadataUpdate) {
buildFetcher();
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0));
- consumerClient.poll(time.timer(0));
- assertEmptyFetch("Should not return records or advance position on fetch error");
- assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()));
- }
- @Test
- public void testFetchFencedLeaderEpoch() {
- buildFetcher();
- assignFromUser(singleton(tp0));
- subscriptions.seek(tp0, 0);
+ final FetchResponse fetchResponse;
- assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0));
- consumerClient.poll(time.timer(0));
+ if (hasTopLevelError)
+ fetchResponse = fetchResponseWithTopLevelError(tidp0, error, 0);
+ else
+ fetchResponse = fullFetchResponse(tidp0, records, error, highWatermark, 0);
+
+ client.prepareResponse(fetchResponse);
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position on fetch error");
- assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update");
- }
- @Test
- public void testFetchUnknownLeaderEpoch() {
- buildFetcher();
- assignFromUser(singleton(tp0));
- subscriptions.seek(tp0, 0);
+ long timeToNextUpdate = metadata.timeToNextUpdate(time.milliseconds());
- assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0));
- consumerClient.poll(time.timer(0));
+ if (shouldRequestMetadataUpdate)
+ assertEquals(0L, timeToNextUpdate, "Should have requested metadata update");
+ else
+ assertNotEquals(0L, timeToNextUpdate, "Should not have requested metadata update");
+ }
- assertEmptyFetch("Should not return records or advance position on fetch error");
- assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update");
+ /**
+ * Supplies parameters to {@link #testHandleFetchResponseError(Errors, long, boolean, boolean)}.
+ */
+ private static Stream<Arguments> handleFetchResponseErrorSupplier() {
+ return Stream.of(
+ Arguments.of(Errors.NOT_LEADER_OR_FOLLOWER, 100L, false, true),
+ Arguments.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, false, true),
+ Arguments.of(Errors.UNKNOWN_TOPIC_ID, -1L, false, true),
+ Arguments.of(Errors.FETCH_SESSION_TOPIC_ID_ERROR, -1L, true, true),
+ Arguments.of(Errors.INCONSISTENT_TOPIC_ID, -1L, false, true),
+ Arguments.of(Errors.FENCED_LEADER_EPOCH, 100L, false, true),
+ Arguments.of(Errors.UNKNOWN_LEADER_EPOCH, 100L, false, false)
+ );
}
@Test
@@ -1636,8 +1585,9 @@ public class FetcherTest {
return false;
}
};
- client.prepareResponse(matcher, fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.pollNoWakeup();
+ client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.pollNoWakeup();
+ assertEquals(0, networkClientDelegate.pendingRequestCount());
}
@Test
@@ -1647,8 +1597,8 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position on fetch error");
assertTrue(subscriptions.isOffsetResetNeeded(tp0));
assertNull(subscriptions.validPosition(tp0));
@@ -1664,9 +1614,9 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
subscriptions.seek(tp0, 1);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position on fetch error");
assertFalse(subscriptions.isOffsetResetNeeded(tp0));
assertEquals(1, subscriptions.position(tp0).offset);
@@ -1681,8 +1631,13 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertTrue(sendFetches() > 0);
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
+
+ // The partition is not marked as needing its offset reset because that error handling logic is
+ // performed during the fetch collection. When we call seek() before we collect the fetch, the
+ // partition's position is updated (to offset 2) which is different from the offset from which
+ // we fetched the data (from offset 0).
assertFalse(subscriptions.isOffsetResetNeeded(tp0));
subscriptions.seek(tp0, 2);
assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition");
@@ -1697,13 +1652,13 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
sendFetches();
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertFalse(subscriptions.isOffsetResetNeeded(tp0));
for (int i = 0; i < 2; i++) {
OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () ->
- fetcher.collectFetch());
+ collectFetch());
assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet());
assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue());
}
@@ -1731,7 +1686,7 @@ public class FetcherTest {
.setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code())
.setHighWatermark(100));
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
List<ConsumerRecord<byte[], byte[]>> allFetchedRecords = new ArrayList<>();
fetchRecordsInto(allFetchedRecords);
@@ -1752,7 +1707,7 @@ public class FetcherTest {
}
private void fetchRecordsInto(List<ConsumerRecord<byte[], byte[]>> allFetchedRecords) {
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
fetchedRecords.values().forEach(allFetchedRecords::addAll);
}
@@ -1792,12 +1747,10 @@ public class FetcherTest {
.setLogStartOffset(0)
.setRecords(partialRecords));
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
List<ConsumerRecord<byte[], byte[]>> fetchedRecords = new ArrayList<>();
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
- for (List<ConsumerRecord<byte[], byte[]>> records : recordsByPartition.values())
- fetchedRecords.addAll(records);
+ fetchRecordsInto(fetchedRecords);
assertEquals(fetchedRecords.size(), subscriptions.position(tp1).offset - 1);
assertEquals(4, subscriptions.position(tp1).offset);
@@ -1805,9 +1758,7 @@ public class FetcherTest {
List<OffsetOutOfRangeException> oorExceptions = new ArrayList<>();
try {
- recordsByPartition = fetchedRecords();
- for (List<ConsumerRecord<byte[], byte[]>> records : recordsByPartition.values())
- fetchedRecords.addAll(records);
+ fetchRecordsInto(fetchedRecords);
} catch (OffsetOutOfRangeException oor) {
oorExceptions.add(oor);
}
@@ -1818,9 +1769,7 @@ public class FetcherTest {
assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0));
assertEquals(oor.offsetOutOfRangePartitions().size(), 1);
- recordsByPartition = fetchedRecords();
- for (List<ConsumerRecord<byte[], byte[]>> records : recordsByPartition.values())
- fetchedRecords.addAll(records);
+ fetchRecordsInto(fetchedRecords);
// Should not have received an Exception for tp2.
assertEquals(6, subscriptions.position(tp2).offset);
@@ -1830,9 +1779,7 @@ public class FetcherTest {
List<KafkaException> kafkaExceptions = new ArrayList<>();
for (int i = 1; i <= numExceptionsExpected; i++) {
try {
- recordsByPartition = fetchedRecords();
- for (List<ConsumerRecord<byte[], byte[]>> records : recordsByPartition.values())
- fetchedRecords.addAll(records);
+ fetchRecordsInto(fetchedRecords);
} catch (KafkaException e) {
kafkaExceptions.add(e);
}
@@ -1851,13 +1798,13 @@ public class FetcherTest {
assertEquals(1, sendFetches());
Map<TopicIdPartition, FetchResponseData.PartitionData> partitions = new HashMap<>();
partitions.put(tidp0, new FetchResponseData.PartitionData()
- .setPartitionIndex(tp0.partition())
- .setHighWatermark(100)
- .setRecords(records));
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ .setPartitionIndex(tp0.partition())
+ .setHighWatermark(100)
+ .setRecords(records));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
- assertEquals(2, fetchedRecords().get(tp0).size());
+ assertEquals(2, fetchRecords().get(tp0).size());
subscriptions.assignFromUser(mkSet(tp0, tp1));
subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1)));
@@ -1865,12 +1812,12 @@ public class FetcherTest {
assertEquals(1, sendFetches());
partitions = new HashMap<>();
partitions.put(tidp1, new FetchResponseData.PartitionData()
- .setPartitionIndex(tp1.partition())
- .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code())
- .setHighWatermark(100));
+ .setPartitionIndex(tp1.partition())
+ .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code())
+ .setHighWatermark(100));
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
- consumerClient.poll(time.timer(0));
- assertEquals(1, fetchedRecords().get(tp0).size());
+ networkClientDelegate.poll(time.timer(0));
+ assertEquals(1, fetchRecords().get(tp0).size());
subscriptions.seek(tp1, 10);
// Should not throw OffsetOutOfRangeException after the seek
@@ -1885,11 +1832,11 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true);
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true);
+ networkClientDelegate.poll(time.timer(0));
assertEmptyFetch("Should not return records or advance position on disconnect");
- // disconnects should have no affect on subscription state
+ // disconnects should have no effect on subscription state
assertFalse(subscriptions.isOffsetResetNeeded(tp0));
assertTrue(subscriptions.isFetchable(tp0));
assertEquals(0, subscriptions.position(tp0).offset);
@@ -1910,7 +1857,7 @@ public class FetcherTest {
time, true, new ApiVersions(), metricsManager.throttleTimeSensor(), new LogContext());
ApiVersionsResponse apiVersionsResponse = TestUtils.defaultApiVersionsResponse(
- 400, ApiMessageType.ListenerType.ZK_BROKER);
+ 400, ApiMessageType.ListenerType.ZK_BROKER);
ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0);
selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
@@ -2116,9 +2063,9 @@ public class FetcherTest {
assertEquals(1, sendFetches());
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertEquals(3, fetchedRecords.get(tp1).size());
assertEquals(3, fetchedRecords.get(tp2).size());
@@ -2130,7 +2077,7 @@ public class FetcherTest {
}
@Test
- public void testFetchResponseMetricsPartialResponse() {
+ public void testFetchResponseMetricsWithSkippedOffset() {
buildFetcher();
assignFromUser(singleton(tp0));
@@ -2188,8 +2135,8 @@ public class FetcherTest {
assertEquals(1, sendFetches());
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
- consumerClient.poll(time.timer(0));
- fetcher.collectFetch();
+ networkClientDelegate.poll(time.timer(0));
+ collectFetch();
int expectedBytes = 0;
for (Record record : records.records())
@@ -2234,8 +2181,8 @@ public class FetcherTest {
.setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes()))));
client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
- consumerClient.poll(time.timer(0));
- fetcher.collectFetch();
+ networkClientDelegate.poll(time.timer(0));
+ collectFetch();
// we should have ignored the record at the wrong offset
int expectedBytes = 0;
@@ -2256,10 +2203,10 @@ public class FetcherTest {
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
// Verify that all metrics except metrics-count have registered templates
@@ -2281,16 +2228,16 @@ public class FetcherTest {
TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) {
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime));
- consumerClient.poll(time.timer(0));
- return fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ return fetchRecords();
}
private Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchRecords(
TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) {
assertEquals(1, sendFetches());
client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime));
- consumerClient.poll(time.timer(0));
- return fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ return fetchRecords();
}
@Test
@@ -2320,7 +2267,7 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
Fetch<byte[], byte[]> fetch = collectFetch();
@@ -2356,10 +2303,10 @@ public class FetcherTest {
return true;
}, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.containsKey(tp0));
assertEquals(fetchedRecords.get(tp0).size(), 2);
}
@@ -2426,10 +2373,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.containsKey(tp0));
// There are only 3 committed records
List<ConsumerRecord<byte[], byte[]>> fetchedConsumerRecords = fetchedRecords.get(tp0);
@@ -2462,7 +2409,7 @@ public class FetcherTest {
buffer.flip();
List<FetchResponseData.AbortedTransaction> abortedTransactions = Collections.singletonList(
- new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)
+ new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)
);
MemoryRecords records = MemoryRecords.readableRecords(buffer);
assignFromUser(singleton(tp0));
@@ -2474,19 +2421,19 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.containsKey(tp0));
assertEquals(fetchedRecords.get(tp0).size(), 2);
List<ConsumerRecord<byte[], byte[]>> fetchedConsumerRecords = fetchedRecords.get(tp0);
- Set<String> committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2"));
+ Set<String> expectedCommittedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2"));
Set<String> actuallyCommittedKeys = new HashSet<>();
for (ConsumerRecord<byte[], byte[]> consumerRecord : fetchedConsumerRecords) {
actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8));
}
- assertEquals(actuallyCommittedKeys, committedKeys);
+ assertEquals(expectedCommittedKeys, actuallyCommittedKeys);
}
@Test
@@ -2515,14 +2462,14 @@ public class FetcherTest {
// prepare the response. the aborted transactions begin at offsets which are no longer in the log
List<FetchResponseData.AbortedTransaction> abortedTransactions = Collections.singletonList(
- new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L));
+ new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L));
client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer),
abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<String, String>>> allFetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<String, String>>> allFetchedRecords = fetchRecords();
assertTrue(allFetchedRecords.containsKey(tp0));
List<ConsumerRecord<String, String>> fetchedRecords = allFetchedRecords.get(tp0);
assertEquals(3, fetchedRecords.size());
@@ -2558,10 +2505,10 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, compactedRecords, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> allFetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> allFetchedRecords = fetchRecords();
assertTrue(allFetchedRecords.containsKey(tp0));
List<ConsumerRecord<byte[], byte[]>> fetchedRecords = allFetchedRecords.get(tp0);
assertEquals(3, fetchedRecords.size());
@@ -2595,7 +2542,7 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
assertEquals(1, sendFetches());
client.prepareResponse(fullFetchResponse(tidp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
Fetch<byte[], byte[]> fetch = collectFetch();
@@ -2653,16 +2600,16 @@ public class FetcherTest {
// prepare the response. the aborted transactions begin at offsets which are no longer in the log
List<FetchResponseData.AbortedTransaction> abortedTransactions = Arrays.asList(
- new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6),
- new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0)
+ new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6),
+ new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0)
);
client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer),
abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<String, String>>> allFetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<String, String>>> allFetchedRecords = fetchRecords();
assertTrue(allFetchedRecords.containsKey(tp0));
List<ConsumerRecord<String, String>> fetchedRecords = allFetchedRecords.get(tp0);
assertEquals(5, fetchedRecords.size());
@@ -2670,7 +2617,7 @@ public class FetcherTest {
}
@Test
- public void testReturnAbortedTransactionsinUncommittedMode() {
+ public void testReturnAbortedTransactionsInUncommittedMode() {
buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(),
new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED);
ByteBuffer buffer = ByteBuffer.allocate(1024);
@@ -2685,7 +2632,7 @@ public class FetcherTest {
buffer.flip();
List<FetchResponseData.AbortedTransaction> abortedTransactions = Collections.singletonList(
- new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0));
+ new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0));
MemoryRecords records = MemoryRecords.readableRecords(buffer);
assignFromUser(singleton(tp0));
@@ -2696,10 +2643,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.containsKey(tp0));
}
@@ -2718,7 +2665,7 @@ public class FetcherTest {
buffer.flip();
List<FetchResponseData.AbortedTransaction> abortedTransactions = Collections.singletonList(
- new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0));
+ new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0));
MemoryRecords records = MemoryRecords.readableRecords(buffer);
assignFromUser(singleton(tp0));
@@ -2729,10 +2676,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
// Ensure that we don't return any of the aborted records, but yet advance the consumer position.
assertFalse(fetchedRecords.containsKey(tp0));
@@ -2743,7 +2690,6 @@ public class FetcherTest {
public void testConsumingViaIncrementalFetchRequests() {
buildFetcher(2);
- List<ConsumerRecord<byte[], byte[]>> records;
assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1)));
subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0)));
subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1)));
@@ -2755,7 +2701,7 @@ public class FetcherTest {
.setHighWatermark(2)
.setLastStableOffset(2)
.setLogStartOffset(0)
- .setRecords(this.records));
+ .setRecords(records));
partitions1.put(tidp1, new FetchResponseData.PartitionData()
.setPartitionIndex(tp1.partition())
.setHighWatermark(100)
@@ -2765,24 +2711,24 @@ public class FetcherTest {
client.prepareResponse(resp1);
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertFalse(fetchedRecords.containsKey(tp1));
- records = fetchedRecords.get(tp0);
- assertEquals(2, records.size());
+ List<ConsumerRecord<byte[], byte[]>> recordsToTest = fetchedRecords.get(tp0);
+ assertEquals(2, recordsToTest.size());
assertEquals(3L, subscriptions.position(tp0).offset);
assertEquals(1L, subscriptions.position(tp1).offset);
- assertEquals(1, records.get(0).offset());
- assertEquals(2, records.get(1).offset());
+ assertEquals(1, recordsToTest.get(0).offset());
+ assertEquals(2, recordsToTest.get(1).offset());
// There is still a buffered record.
assertEquals(0, sendFetches());
- fetchedRecords = fetchedRecords();
+ fetchedRecords = fetchRecords();
assertFalse(fetchedRecords.containsKey(tp1));
- records = fetchedRecords.get(tp0);
- assertEquals(1, records.size());
- assertEquals(3, records.get(0).offset());
+ recordsToTest = fetchedRecords.get(tp0);
+ assertEquals(1, recordsToTest.size());
+ assertEquals(3, recordsToTest.get(0).offset());
assertEquals(4L, subscriptions.position(tp0).offset);
// The second response contains no new records.
@@ -2790,8 +2736,8 @@ public class FetcherTest {
FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2);
client.prepareResponse(resp2);
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- fetchedRecords = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.isEmpty());
assertEquals(4L, subscriptions.position(tp0).offset);
assertEquals(1L, subscriptions.position(tp1).offset);
@@ -2803,225 +2749,19 @@ public class FetcherTest {
.setHighWatermark(100)
.setLastStableOffset(4)
.setLogStartOffset(0)
- .setRecords(this.nextRecords));
+ .setRecords(nextRecords));
FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3);
client.prepareResponse(resp3);
assertEquals(1, sendFetches());
- consumerClient.poll(time.timer(0));
- fetchedRecords = fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ fetchedRecords = fetchRecords();
assertFalse(fetchedRecords.containsKey(tp1));
- records = fetchedRecords.get(tp0);
- assertEquals(2, records.size());
+ recordsToTest = fetchedRecords.get(tp0);
+ assertEquals(2, recordsToTest.size());
assertEquals(6L, subscriptions.position(tp0).offset);
assertEquals(1L, subscriptions.position(tp1).offset);
- assertEquals(4, records.get(0).offset());
- assertEquals(5, records.get(1).offset());
- }
-
- @Test
- public void testFetcherConcurrency() throws Exception {
- int numPartitions = 20;
- Set<TopicPartition> topicPartitions = new HashSet<>();
- for (int i = 0; i < numPartitions; i++)
- topicPartitions.add(new TopicPartition(topicName, i));
-
- LogContext logContext = new LogContext();
- buildDependencies(new MetricConfig(), Long.MAX_VALUE, new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST), logContext);
-
- IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED;
-
- offsetFetcher = new OffsetFetcher(logContext,
- consumerClient,
- metadata,
- subscriptions,
- time,
- retryBackoffMs,
- requestTimeoutMs,
- isolationLevel,
- apiVersions);
-
- FetchConfig<byte[], byte[]> fetchConfig = new FetchConfig<>(
- minBytes,
- maxBytes,
- maxWaitMs,
- fetchSize,
- 2 * numPartitions,
- true, // check crcs
- CommonClientConfigs.DEFAULT_CLIENT_RACK,
- new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()),
- isolationLevel);
- fetcher = new Fetcher<byte[], byte[]>(
- logContext,
- consumerClient,
- metadata,
- subscriptions,
- fetchConfig,
- metricsManager,
- time) {
- @Override
- protected FetchSessionHandler sessionHandler(int node) {
- final FetchSessionHandler handler = super.sessionHandler(node);
- if (handler == null)
- return null;
- else {
- return new FetchSessionHandler(new LogContext(), node) {
- @Override
- public Builder newBuilder() {
- verifySessionPartitions();
- return handler.newBuilder();
- }
-
- @Override
- public boolean handleResponse(FetchResponse response, short version) {
- verifySessionPartitions();
- return handler.handleResponse(response, version);
- }
-
- @Override
- public void handleError(Throwable t) {
- verifySessionPartitions();
- handler.handleError(t);
- }
-
- // Verify that session partitions can be traversed safely.
- private void verifySessionPartitions() {
- try {
- Field field = FetchSessionHandler.class.getDeclaredField("sessionPartitions");
- field.setAccessible(true);
- LinkedHashMap<?, ?> sessionPartitions =
- (LinkedHashMap<?, ?>) field.get(handler);
- for (Map.Entry<?, ?> entry : sessionPartitions.entrySet()) {
- // If `sessionPartitions` are modified on another thread, Thread.yield will increase the
- // possibility of ConcurrentModificationException if appropriate synchronization is not used.
- Thread.yield();
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
- }
- };
-
- MetadataResponse initialMetadataResponse = RequestTestUtils.metadataUpdateWithIds(1,
- singletonMap(topicName, numPartitions), tp -> validLeaderEpoch, topicIds);
- client.updateMetadata(initialMetadataResponse);
- fetchSize = 10000;
-
- assignFromUser(topicPartitions);
- topicPartitions.forEach(tp -> subscriptions.seek(tp, 0L));
-
- AtomicInteger fetchesRemaining = new AtomicInteger(400);
- executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(() -> {
- while (fetchesRemaining.get() > 0) {
- synchronized (consumerClient) {
- if (!client.requests().isEmpty()) {
- ClientRequest request = client.requests().peek();
- FetchRequest fetchRequest = (FetchRequest) request.requestBuilder().build();
- LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> responseMap = new LinkedHashMap<>();
- for (Map.Entry<TopicIdPartition, FetchRequest.PartitionData> entry : fetchRequest.fetchData(topicNames).entrySet()) {
- TopicIdPartition tp = entry.getKey();
- long offset = entry.getValue().fetchOffset;
- responseMap.put(tp, new FetchResponseData.PartitionData()
- .setPartitionIndex(tp.topicPartition().partition())
- .setHighWatermark(offset + 2)
- .setLastStableOffset(offset + 2)
- .setLogStartOffset(0)
- .setRecords(buildRecords(offset, 2, offset)));
- }
- client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap));
- consumerClient.poll(time.timer(0));
- }
- }
- }
- return fetchesRemaining.get();
- });
- Map<TopicPartition, Long> nextFetchOffsets = topicPartitions.stream()
- .collect(Collectors.toMap(Function.identity(), t -> 0L));
- while (fetchesRemaining.get() > 0 && !future.isDone()) {
- if (sendFetches() == 1) {
- synchronized (consumerClient) {
- consumerClient.poll(time.timer(0));
- }
- }
- if (fetcher.hasCompletedFetches()) {
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
- if (!fetchedRecords.isEmpty()) {
- fetchesRemaining.decrementAndGet();
- fetchedRecords.forEach((tp, records) -> {
- assertEquals(2, records.size());
- long nextOffset = nextFetchOffsets.get(tp);
- assertEquals(nextOffset, records.get(0).offset());
- assertEquals(nextOffset + 1, records.get(1).offset());
- nextFetchOffsets.put(tp, nextOffset + 2);
- });
- }
- }
- }
- assertEquals(0, future.get());
- }
-
- @Test
- public void testFetcherSessionEpochUpdate() throws Exception {
- buildFetcher(2);
-
- MetadataResponse initialMetadataResponse = RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, 1), topicIds);
- client.updateMetadata(initialMetadataResponse);
- assignFromUser(Collections.singleton(tp0));
- subscriptions.seek(tp0, 0L);
-
- AtomicInteger fetchesRemaining = new AtomicInteger(1000);
- executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(() -> {
- long nextOffset = 0;
- long nextEpoch = 0;
- while (fetchesRemaining.get() > 0) {
- synchronized (consumerClient) {
- if (!client.requests().isEmpty()) {
- ClientRequest request = client.requests().peek();
- FetchRequest fetchRequest = (FetchRequest) request.requestBuilder().build();
- int epoch = fetchRequest.metadata().epoch();
- assertTrue(epoch == 0 || epoch == nextEpoch,
- String.format("Unexpected epoch expected %d got %d", nextEpoch, epoch));
- nextEpoch++;
- LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> responseMap = new LinkedHashMap<>();
- responseMap.put(tidp0, new FetchResponseData.PartitionData()
- .setPartitionIndex(tp0.partition())
- .setHighWatermark(nextOffset + 2)
- .setLastStableOffset(nextOffset + 2)
- .setLogStartOffset(0)
- .setRecords(buildRecords(nextOffset, 2, nextOffset)));
- nextOffset += 2;
- client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap));
- consumerClient.poll(time.timer(0));
- }
- }
- }
- return fetchesRemaining.get();
- });
- long nextFetchOffset = 0;
- while (fetchesRemaining.get() > 0 && !future.isDone()) {
- if (sendFetches() == 1) {
- synchronized (consumerClient) {
- consumerClient.poll(time.timer(0));
- }
- }
- if (fetcher.hasCompletedFetches()) {
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
- if (!fetchedRecords.isEmpty()) {
- fetchesRemaining.decrementAndGet();
- List<ConsumerRecord<byte[], byte[]>> records = fetchedRecords.get(tp0);
- assertEquals(2, records.size());
- assertEquals(nextFetchOffset, records.get(0).offset());
- assertEquals(nextFetchOffset + 1, records.get(1).offset());
- nextFetchOffset += 2;
- }
- assertTrue(fetchedRecords().isEmpty());
- }
- }
- assertEquals(0, future.get());
+ assertEquals(4, recordsToTest.get(0).offset());
+ assertEquals(5, recordsToTest.get(1).offset());
}
@Test
@@ -3058,10 +2798,10 @@ public class FetcherTest {
return true;
}, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchRecords();
assertTrue(fetchedRecords.containsKey(tp0));
assertEquals(fetchedRecords.get(tp0).size(), 2);
}
@@ -3142,88 +2882,10 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
- consumerClient.pollNoWakeup();
+ networkClientDelegate.pollNoWakeup();
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
- assertTrue(partitionRecords.containsKey(tp0));
-
- assertEquals(subscriptions.position(tp0).offset, 3L);
- assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1));
- }
-
- @Test
- public void testTruncationDetected() {
- // Create some records that include a leader epoch (1)
- MemoryRecordsBuilder builder = MemoryRecords.builder(
- ByteBuffer.allocate(1024),
- RecordBatch.CURRENT_MAGIC_VALUE,
- CompressionType.NONE,
- TimestampType.CREATE_TIME,
- 0L,
- RecordBatch.NO_TIMESTAMP,
- RecordBatch.NO_PRODUCER_ID,
- RecordBatch.NO_PRODUCER_EPOCH,
- RecordBatch.NO_SEQUENCE,
- false,
- 1 // record epoch is earlier than the leader epoch on the client
- );
- builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes());
- builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes());
- builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes());
- MemoryRecords records = builder.build();
-
- buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(),
- Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED);
- assignFromUser(singleton(tp0));
-
- // Initialize the epoch=2
- Map<String, Integer> partitionCounts = new HashMap<>();
- partitionCounts.put(tp0.topic(), 4);
- MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(),
- partitionCounts, tp -> 2, topicIds);
- metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L);
-
- // Offset validation requires OffsetForLeaderEpoch request v3 or higher
- Node node = metadata.fetch().nodes().get(0);
- apiVersions.update(node.idString(), NodeApiVersions.create());
-
- // Seek
- Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.currentLeader(tp0).leader, Optional.of(1));
- subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(1), leaderAndEpoch));
-
- // Check for truncation, this should cause tp0 to go into validation
- OffsetFetcher offsetFetcher = new OffsetFetcher(new LogContext(),
- consumerClient,
- metadata,
- subscriptions,
- time,
- retryBackoffMs,
- requestTimeoutMs,
- IsolationLevel.READ_UNCOMMITTED,
- apiVersions);
- offsetFetcher.validatePositionsIfNeeded();
-
- // No fetches sent since we entered validation
- assertEquals(0, sendFetches());
- assertFalse(fetcher.hasCompletedFetches());
- assertTrue(subscriptions.awaitingValidation(tp0));
-
- // Prepare OffsetForEpoch response then check that we update the subscription position correctly.
- client.prepareResponse(prepareOffsetsForLeaderEpochResponse(tp0, Errors.NONE, 1, 10L));
- consumerClient.pollNoWakeup();
-
- assertFalse(subscriptions.awaitingValidation(tp0));
-
- // Fetch again, now it works
- assertEquals(1, sendFetches());
- assertFalse(fetcher.hasCompletedFetches());
-
- client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
- consumerClient.pollNoWakeup();
- assertTrue(fetcher.hasCompletedFetches());
-
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
assertEquals(subscriptions.position(tp0).offset, 3L);
@@ -3239,7 +2901,7 @@ public class FetcherTest {
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false));
subscriptions.seek(tp0, 0);
- // Node preferred replica before first fetch response
+ // Take note of the preferred replica before the first fetch response
Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(-1, selected.id());
@@ -3247,12 +2909,12 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
// Set preferred read replica to node=1
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
// Verify
@@ -3264,11 +2926,11 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
// Set preferred read replica to node=2, which isn't in our metadata, should revert to leader
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(-1, selected.id());
}
@@ -3284,11 +2946,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
// Set preferred read replica to node=1
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Verify
Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
@@ -3297,11 +2959,11 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
// Disconnect - preferred read replica should be cleared.
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true);
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertFalse(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(-1, selected.id());
}
@@ -3309,7 +2971,7 @@ public class FetcherTest {
@Test
public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned() {
buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(),
- Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis());
+ Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis());
subscriptions.assignFromUser(singleton(tp0));
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false));
@@ -3317,11 +2979,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
// Set preferred read replica to node=1
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
- FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
+ FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Verify
Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
@@ -3330,13 +2992,13 @@ public class FetcherTest {
assertFalse(fetcher.hasCompletedFetches());
// Disconnect and remove tp0 from assignment
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true);
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true);
subscriptions.assignFromUser(emptySet());
// Preferred read replica should not be cleared
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertFalse(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(-1, selected.id());
}
@@ -3352,11 +3014,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
// Set preferred read replica to node=1
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Verify
Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
@@ -3369,9 +3031,9 @@ public class FetcherTest {
client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.EMPTY, Errors.NOT_LEADER_OR_FOLLOWER, -1L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(-1, selected.id());
}
@@ -3389,26 +3051,26 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L,
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(selected.id(), 1);
- // Return an error, should unset the preferred read replica
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L,
+ // Return an error, should unset the preferred read replica
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L,
FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty()));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
assertEquals(selected.id(), -1);
@@ -3421,8 +3083,8 @@ public class FetcherTest {
subscriptions.seek(tp0, 0);
sendFetches();
client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0));
- consumerClient.poll(time.timer(0));
- fetchedRecords();
+ networkClientDelegate.poll(time.timer(0));
+ fetchRecords();
Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader;
assertTrue(leaderAndEpoch.leader.isPresent());
@@ -3431,15 +3093,15 @@ public class FetcherTest {
AtomicBoolean wokenUp = new AtomicBoolean(false);
client.setWakeupHook(() -> {
if (!wokenUp.getAndSet(true)) {
- consumerClient.disconnectAsync(readReplica);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.disconnectAsync(readReplica);
+ networkClientDelegate.poll(time.timer(0));
}
});
assertEquals(1, sendFetches());
- consumerClient.disconnectAsync(readReplica);
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.disconnectAsync(readReplica);
+ networkClientDelegate.poll(time.timer(0));
assertEquals(1, sendFetches());
}
@@ -3459,27 +3121,27 @@ public class FetcherTest {
buildRecords(1L, 1, 1),
Errors.CORRUPT_MESSAGE,
100L, 0));
- consumerClient.poll(time.timer(0));
+ networkClientDelegate.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
// Trigger the exception.
- assertThrows(KafkaException.class, this::fetchedRecords);
+ assertThrows(KafkaException.class, this::fetchRecords);
}
private OffsetsForLeaderEpochResponse prepareOffsetsForLeaderEpochResponse(
- TopicPartition topicPartition,
- Errors error,
- int leaderEpoch,
- long endOffset
+ TopicPartition topicPartition,
+ Errors error,
+ int leaderEpoch,
+ long endOffset
) {
OffsetForLeaderEpochResponseData data = new OffsetForLeaderEpochResponseData();
data.topics().add(new OffsetForLeaderTopicResult()
- .setTopic(topicPartition.topic())
- .setPartitions(Collections.singletonList(new EpochEndOffset()
- .setPartition(topicPartition.partition())
- .setErrorCode(error.code())
- .setLeaderEpoch(leaderEpoch)
- .setEndOffset(endOffset))));
+ .setTopic(topicPartition.topic())
+ .setPartitions(Collections.singletonList(new EpochEndOffset()
+ .setPartition(topicPartition.partition())
+ .setErrorCode(error.code())
+ .setLeaderEpoch(leaderEpoch)
+ .setEndOffset(endOffset))));
return new OffsetsForLeaderEpochResponse(data);
}
@@ -3577,14 +3239,14 @@ public class FetcherTest {
assertTrue(fetch.isEmpty(), reason);
}
- private <K, V> Map<TopicPartition, List<ConsumerRecord<K, V>>> fetchedRecords() {
+ private <K, V> Map<TopicPartition, List<ConsumerRecord<K, V>>> fetchRecords() {
Fetch<K, V> fetch = collectFetch();
return fetch.records();
}
@SuppressWarnings("unchecked")
private <K, V> Fetch<K, V> collectFetch() {
- return (Fetch) fetcher.collectFetch();
+ return (Fetch<K, V>) fetcher.collectFetch();
}
private void buildFetcher(int maxPollRecords) {
@@ -3642,7 +3304,8 @@ public class FetcherTest {
SubscriptionState subscriptionState,
LogContext logContext) {
buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext);
- FetchConfig<K, V> fetchConfig = new FetchConfig<>(
+ Deserializers<K, V> deserializers = new Deserializers<>(keyDeserializer, valueDeserializer);
+ FetchConfig fetchConfig = new FetchConfig(
minBytes,
maxBytes,
maxWaitMs,
@@ -3650,18 +3313,34 @@ public class FetcherTest {
maxPollRecords,
true, // check crc
CommonClientConfigs.DEFAULT_CLIENT_RACK,
- new Deserializers<>(keyDeserializer, valueDeserializer),
isolationLevel);
- fetcher = spy(new Fetcher<>(
+ FetchCollector<K, V> fetchCollector = new FetchCollector<>(logContext,
+ metadata,
+ subscriptions,
+ fetchConfig,
+ deserializers,
+ metricsManager,
+ time);
+ fetcher = spy(new TestableFetchRequestManager<>(
logContext,
- consumerClient,
+ time,
metadata,
subscriptionState,
fetchConfig,
+ new FetchBuffer(logContext),
metricsManager,
- time));
+ networkClientDelegate,
+ fetchCollector));
+ ConsumerNetworkClient consumerNetworkClient = new ConsumerNetworkClient(
+ logContext,
+ client,
+ metadata,
+ time,
+ retryBackoffMs,
+ (int) requestTimeoutMs,
+ Integer.MAX_VALUE);
offsetFetcher = new OffsetFetcher(logContext,
- consumerClient,
+ consumerNetworkClient,
metadata,
subscriptions,
time,
@@ -3675,19 +3354,159 @@ public class FetcherTest {
long metadataExpireMs,
SubscriptionState subscriptionState,
LogContext logContext) {
- time = new MockTime(1);
+ time = new MockTime(1, 0, 0);
subscriptions = subscriptionState;
metadata = new ConsumerMetadata(0, 0, metadataExpireMs, false, false,
subscriptions, logContext, new ClusterResourceListeners());
client = new MockClient(time, metadata);
metrics = new Metrics(metricConfig, time);
- consumerClient = spy(new ConsumerNetworkClient(logContext, client, metadata, time,
- 100, 1000, Integer.MAX_VALUE));
metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumer" + groupId);
metricsManager = new FetchMetricsManager(metrics, metricsRegistry);
+
+ Properties properties = new Properties();
+ properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+ properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
+ properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs));
+ properties.setProperty(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs));
+ ConsumerConfig config = new ConsumerConfig(properties);
+ networkClientDelegate = spy(new TestableNetworkClientDelegate(time, config, logContext, client));
}
private <T> List<Long> collectRecordOffsets(List<ConsumerRecord<T, T>> records) {
return records.stream().map(ConsumerRecord::offset).collect(Collectors.toList());
}
+
+ private class TestableFetchRequestManager<K, V> extends FetchRequestManager {
+
+ private final FetchCollector<K, V> fetchCollector;
+
+ public TestableFetchRequestManager(LogContext logContext,
+ Time time,
+ ConsumerMetadata metadata,
+ SubscriptionState subscriptions,
+ FetchConfig fetchConfig,
+ FetchBuffer fetchBuffer,
+ FetchMetricsManager metricsManager,
+ NetworkClientDelegate networkClientDelegate,
+ FetchCollector<K, V> fetchCollector) {
+ super(logContext, time, metadata, subscriptions, fetchConfig, fetchBuffer, metricsManager, networkClientDelegate);
+ this.fetchCollector = fetchCollector;
+ }
+
+ private Fetch<K, V> collectFetch() {
+ return fetchCollector.collectFetch(fetchBuffer);
+ }
+
+ private int sendFetches() {
+ NetworkClientDelegate.PollResult pollResult = poll(time.milliseconds());
+ networkClientDelegate.addAll(pollResult.unsentRequests);
+ return pollResult.unsentRequests.size();
+ }
+
+ private void clearBufferedDataForUnassignedPartitions(Set<TopicPartition> partitions) {
+ fetchBuffer.retainAll(partitions);
+ }
+ }
+
+ private class TestableNetworkClientDelegate extends NetworkClientDelegate {
+
+ private final Logger log = LoggerFactory.getLogger(NetworkClientDelegate.class);
+ private final ConcurrentLinkedQueue<Node> pendingDisconnects = new ConcurrentLinkedQueue<>();
+
+ public TestableNetworkClientDelegate(Time time,
+ ConsumerConfig config,
+ LogContext logContext,
+ KafkaClient client) {
+ super(time, config, logContext, client);
+ }
+
+ @Override
+ public void poll(final long timeoutMs, final long currentTimeMs) {
+ handlePendingDisconnects();
+ super.poll(timeoutMs, currentTimeMs);
+ }
+
+ public void pollNoWakeup() {
+ poll(time.timer(0));
+ }
+
+ public int pendingRequestCount() {
+ return unsentRequests().size() + client.inFlightRequestCount();
+ }
+
+ public void poll(final Timer timer) {
+ long pollTimeout = Math.min(timer.remainingMs(), requestTimeoutMs);
+ if (client.inFlightRequestCount() == 0)
+ pollTimeout = Math.min(pollTimeout, retryBackoffMs);
+ poll(pollTimeout, timer.currentTimeMs());
+ }
+
+ private Set<Node> unsentRequestNodes() {
+ Set<Node> set = new HashSet<>();
+
+ for (UnsentRequest u : unsentRequests())
+ u.node().ifPresent(set::add);
+
+ return set;
+ }
+
+ private List<UnsentRequest> removeUnsentRequestByNode(Node node) {
+ List<UnsentRequest> list = new ArrayList<>();
+
+ Iterator<UnsentRequest> iter = unsentRequests().iterator();
+
+ while (iter.hasNext()) {
+ UnsentRequest u = iter.next();
+
+ if (node.equals(u.node().orElse(null))) {
+ iter.remove();
+ list.add(u);
+ }
+ }
+
+ return list;
+ }
+
+ @Override
+ protected void checkDisconnects(final long currentTimeMs) {
+ // any disconnects affecting requests that have already been transmitted will be handled
+ // by NetworkClient, so we just need to check whether connections for any of the unsent
+ // requests have been disconnected; if they have, then we complete the corresponding future
+ // and set the disconnect flag in the ClientResponse
+ for (Node node : unsentRequestNodes()) {
+ if (client.connectionFailed(node)) {
+ // Remove entry before invoking request callback to avoid callbacks handling
+ // coordinator failures traversing the unsent list again.
+ for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) {
+ // TODO: this should likely emulate what's done in ConsumerNetworkClient
+ log.error("checkDisconnects - please update! unsentRequest: {}", unsentRequest);
+ }
+ }
+ }
+ }
+
+ private void handlePendingDisconnects() {
+ while (true) {
+ Node node = pendingDisconnects.poll();
+ if (node == null)
+ break;
+
+ failUnsentRequests(node, DisconnectException.INSTANCE);
+ client.disconnect(node.idString());
+ }
+ }
+
+ public void disconnectAsync(Node node) {
+ pendingDisconnects.offer(node);
+ client.wakeup();
+ }
+
+ private void failUnsentRequests(Node node, RuntimeException e) {
+ // clear unsent requests to node and fail their corresponding futures
+ for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) {
+ FutureCompletionHandler handler = unsentRequest.handler();
+ handler.onFailure(time.milliseconds(), e);
+ }
+ }
+ }
}
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index 6336c11219e..a66d153f098 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -90,6 +90,9 @@ import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import java.io.DataOutputStream;
@@ -116,6 +119,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
@@ -209,22 +213,22 @@ public class FetcherTest {
tp -> validLeaderEpoch, topicIds), false, 0L);
}
- private void assignFromUserNoId(Set<TopicPartition> partitions) {
- subscriptions.assignFromUser(partitions);
- client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap()));
+ private void assignFromUser(TopicPartition partition) {
+ subscriptions.assignFromUser(singleton(partition));
+ client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap(partition.topic(), 1), Collections.emptyMap()));
// A dummy metadata update to ensure valid leader epoch.
metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1,
- Collections.emptyMap(), singletonMap("noId", 1),
+ Collections.emptyMap(), singletonMap(partition.topic(), 1),
tp -> validLeaderEpoch, topicIds), false, 0L);
}
@AfterEach
public void teardown() throws Exception {
if (metrics != null)
- this.metrics.close();
+ metrics.close();
if (fetcher != null)
- this.fetcher.close();
+ fetcher.close();
if (executorService != null) {
executorService.shutdownNow();
assertTrue(executorService.awaitTermination(5, TimeUnit.SECONDS));
@@ -247,11 +251,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp0);
@@ -274,9 +278,9 @@ public class FetcherTest {
assertEquals(1, sendFetches());
subscriptions.markPendingRevocation(singleton(tp0));
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
consumerClient.poll(time.timer(0));
- assertNull(fetchedRecords().get(tp0));
+ assertNull(fetchRecords().get(tp0));
}
@Test
@@ -301,7 +305,7 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0);
+ final FetchResponse fetchResponse = fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0);
client.prepareResponse(fetchResponse);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
@@ -310,7 +314,7 @@ public class FetcherTest {
final ArgumentCaptor<FetchRequest.Builder> argument = ArgumentCaptor.forClass(FetchRequest.Builder.class);
// send request to close the fetcher
- this.fetcher.close(time.timer(Duration.ofSeconds(10)));
+ fetcher.close(time.timer(Duration.ofSeconds(10)));
// validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal
// fetch earlier and another for the finish fetch here.
@@ -332,10 +336,10 @@ public class FetcherTest {
// normal fetch
assertEquals(1, sendFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position
// mark partition unfetchable
@@ -343,7 +347,7 @@ public class FetcherTest {
assertEquals(0, sendFetches());
consumerClient.poll(time.timer(0));
assertFalse(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
assertEquals(4L, subscriptions.position(tp0).offset);
}
@@ -353,21 +357,21 @@ public class FetcherTest {
buildFetcher();
TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0));
- assignFromUserNoId(singleton(noId.topicPartition()));
+ assignFromUser(noId.topicPartition());
subscriptions.seek(noId.topicPartition(), 0);
- // Fetch should use request version 12
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
+ // Fetch should use request version 12
client.prepareResponse(
fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)),
- fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(noId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(noId.topicPartition()));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(noId.topicPartition());
@@ -388,18 +392,18 @@ public class FetcherTest {
assignFromUser(singleton(tp.topicPartition()));
subscriptions.seek(tp.topicPartition(), 0);
- // Fetch should use latest version
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
+ // Fetch should use latest version
client.prepareResponse(
fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)),
- fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(tp, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp.topicPartition()));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp.topicPartition());
@@ -424,9 +428,9 @@ public class FetcherTest {
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch));
subscriptions.seek(foo.topicPartition(), 0);
- // Fetch should use latest version.
assertEquals(1, sendFetches());
+ // Fetch should use latest version.
client.prepareResponse(
fetchRequestMatcher(ApiKeys.FETCH.latestVersion(),
singletonMap(foo, new PartitionData(
@@ -438,13 +442,13 @@ public class FetcherTest {
),
emptyList()
),
- fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, foo, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
- // Assign bar and unassign foo.
+ // Assign bar and un-assign foo.
subscriptions.assignFromUser(singleton(bar.topicPartition()));
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch));
subscriptions.seek(bar.topicPartition(), 0);
@@ -464,11 +468,11 @@ public class FetcherTest {
),
singletonList(foo)
),
- fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, bar, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
@Test
@@ -497,11 +501,11 @@ public class FetcherTest {
),
emptyList()
),
- fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithOldTopicId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Replace foo with old topic id with foo with new topic id.
subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition()));
@@ -524,11 +528,11 @@ public class FetcherTest {
),
singletonList(fooWithOldTopicId)
),
- fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithNewTopicId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
@Test
@@ -536,7 +540,6 @@ public class FetcherTest {
buildFetcher();
TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0));
- TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
// Assign foo without a topic id.
subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition()));
@@ -557,13 +560,14 @@ public class FetcherTest {
),
emptyList()
),
- fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Upgrade.
+ TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
subscriptions.assignFromUser(singleton(fooWithId.topicPartition()));
client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch));
subscriptions.seek(fooWithId.topicPartition(), 0);
@@ -584,11 +588,11 @@ public class FetcherTest {
),
emptyList()
),
- fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
// Downgrade.
subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition()));
@@ -611,11 +615,11 @@ public class FetcherTest {
),
emptyList()
),
- fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0)
+ fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0)
);
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- fetchedRecords();
+ fetchRecords();
}
private MockClient.RequestMatcher fetchRequestMatcher(
@@ -684,7 +688,7 @@ public class FetcherTest {
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
assertEquals(2, partitionRecords.get(tp0).size());
@@ -735,7 +739,7 @@ public class FetcherTest {
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
assertEquals(6, partitionRecords.get(tp0).size());
@@ -756,7 +760,7 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
Set<TopicPartition> newAssignedTopicPartitions = new HashSet<>();
@@ -812,7 +816,7 @@ public class FetcherTest {
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertTrue(partitionRecords.containsKey(tp0));
List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(tp0);
@@ -833,11 +837,11 @@ public class FetcherTest {
assertEquals(1, sendFetches());
assertFalse(fetcher.hasCompletedFetches());
- client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0));
+ client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0));
consumerClient.poll(time.timer(0));
assertTrue(fetcher.hasCompletedFetches());
- Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchRecords();
assertFalse(partitionRecords.containsKey(tp0));
}
@@ -872,19 +876,16 @@ public class FetcherTest {
assignFromUser(singleton(tp0));
subscriptions.seek(tp0, 1);
- client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
+ client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
assertEquals(1, sendFetches());
consumerClient.poll(time.timer(0));
- // The fetcher should block on Deserialization error
+
for (int i = 0; i < 2; i++) {
- try {
- fetcher.collectFetch();
- fail("fetchedRecords should have raised");
- } catch (SerializationException e) {
- // the position should not advance since no data has been returned
- assertEquals(1, subscriptions.position(tp0).offset);
- }
... 2375 lines suppressed ...