You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2026/09/03 17:48:46 UTC
(kafka) branch 4.4 updated: KAFKA-20970 Another busy loop happens if auto.commit.interval.ms is less than bootstrap.resolve.timeout.ms (#23227)
This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch 4.4
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/4.4 by this push:
new cd44c5de0ff KAFKA-20970 Another busy loop happens if auto.commit.interval.ms is less than bootstrap.resolve.timeout.ms (#23227)
cd44c5de0ff is described below
commit cd44c5de0ff7b9f7067619a5b56c1f2af87f6030
Author: Ken Huang <s7...@gmail.com>
AuthorDate: Thu Sep 3 22:57:46 2026 +0800
KAFKA-20970 Another busy loop happens if auto.commit.interval.ms is less than bootstrap.resolve.timeout.ms (#23227)
`CommitRequestManager#maximumTimeToWait()` returned the raw remaining
time of the auto-commit timer, which becomes `0` once the interval
elapses. However, `poll()` cannot actually send an auto-commit until
the coordinator is known; it returns `EMPTY` while
`coordinatorRequestManager.coordinator()` is empty. When bootstrap DNS
resolution is still in progress (`bootstrap.resolve.timeout.ms > 0`) and
`auto.commit.interval.ms` is shorter than the resolution window, the
auto-commit timer can expire before the coordinator is discovered.
`remainingMs()` then continues to return `0` indefinitely, since the
timer is not reset until the application thread successfully sends an
auto-commit.
This `0` is propagated to
`ConsumerNetworkThread#cachedMaximumTimeToWait`, which bounds the
`pollTimeout` in `AsyncKafkaConsumer#pollForFetches()` to `0`, causing
the application and network threads to busy-loop for as long as the
coordinator remains unknown.
Reviewers: Chia-Ping Tsai <ch...@gmail.com>, Andrew Schofield
<as...@confluent.io>, TengYao Chi <fr...@apache.org>, Lianet
Magrans <lm...@confluent.io>
---
checkstyle/suppressions.xml | 4 +-
.../consumer/internals/CommitRequestManager.java | 18 ++++-
.../StreamsGroupHeartbeatRequestManager.java | 16 +++-
.../internals/CommitRequestManagerTest.java | 87 ++++++++++++++++++++++
.../StreamsGroupHeartbeatRequestManagerTest.java | 21 ++++++
5 files changed, 140 insertions(+), 6 deletions(-)
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 549743ef46e..d003eca5c93 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -86,10 +86,10 @@
<!-- Clients tests -->
<suppress checks="ClassDataAbstractionCoupling"
- files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
+ files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer|CommitRequestManager)Test.java"/>
<suppress checks="ClassFanOutComplexity"
- files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer|NetworkClient|StreamsGroupHeartbeatRequestManager)Test.java"/>
+ files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer|NetworkClient|StreamsGroupHeartbeatRequestManager|CommitRequestManager)Test.java"/>
<!-- KafkaAdminClient test split: shared base + per-domain test classes -->
<suppress checks="(ClassDataAbstractionCoupling|ClassFanOutComplexity|JavaNCSS)"
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
index a287f935799..1324ece542f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java
@@ -221,7 +221,19 @@ public class CommitRequestManager implements RequestManager, MemberStateListener
*/
@Override
public long maximumTimeToWait(long currentTimeMs) {
- return autoCommitState.map(ac -> ac.remainingMs(currentTimeMs)).orElse(Long.MAX_VALUE);
+ if (autoCommitState.isEmpty()) {
+ return Long.MAX_VALUE;
+ }
+ AutoCommitState autoCommit = autoCommitState.get();
+ // An auto-commit is only sent when the coordinator is known; poll() returns EMPTY otherwise.
+ // If the coordinator is unavailable (e.g. bootstrap DNS resolution is still in progress),
+ // falling through to the timer-based remainingMs() would return 0 once the auto-commit interval
+ // elapses, since the auto-commit timer remains permanently expired. This would cause both the
+ // application and network threads to busy-spin.
+ if (coordinatorRequestManager.coordinator().isEmpty()) {
+ return autoCommit.autoCommitIntervalMs();
+ }
+ return autoCommit.remainingMs(currentTimeMs);
}
private static long findMinTime(final Collection<? extends RequestState> requests, final long currentTimeMs) {
@@ -1564,6 +1576,10 @@ public class CommitRequestManager implements RequestManager, MemberStateListener
public void setInflightCommitStatus(final boolean inflightCommitStatus) {
this.hasInflightCommit = inflightCommitStatus;
}
+
+ public long autoCommitIntervalMs() {
+ return autoCommitInterval;
+ }
}
static class MemberInfo {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManager.java
index f9236be57f7..580cd9289e9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManager.java
@@ -530,9 +530,19 @@ public class StreamsGroupHeartbeatRequestManager implements RequestManager {
@Override
public long maximumTimeToWait(long currentTimeMs) {
pollTimer.update(currentTimeMs);
- if (pollTimer.isExpired() ||
- membershipManager.shouldNotWaitForHeartbeatInterval() && !heartbeatRequestState.requestInFlight()) {
-
+ if (pollTimer.isExpired()) {
+ return 0L;
+ }
+ // A heartbeat is only sent when the coordinator is known; poll() returns EMPTY otherwise
+ // (see the guard at the top of poll()). If the coordinator is unavailable (for example,
+ // while bootstrap DNS resolution is still in progress), the
+ // shouldNotWaitForHeartbeatInterval() check would return 0 whenever the member wants to
+ // (re)join. Because no heartbeat can be sent until the coordinator is discovered, the
+ // condition remains true and both the application and network threads end up busy-spinning.
+ if (coordinatorRequestManager.coordinator().isEmpty() || membershipManager.shouldSkipHeartbeat()) {
+ return heartbeatRequestState.heartbeatIntervalMs();
+ }
+ if (membershipManager.shouldNotWaitForHeartbeatInterval() && !heartbeatRequestState.requestInFlight()) {
return 0L;
}
return Math.min(pollTimer.remainingMs() / 2, heartbeatRequestState.timeToNextHeartbeatMs(currentTimeMs));
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
index 426b55833ed..33863fcd765 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java
@@ -16,15 +16,23 @@
*/
package org.apache.kafka.clients.consumer.internals;
+import org.apache.kafka.clients.ApiVersions;
+import org.apache.kafka.clients.BootstrapConfiguration;
+import org.apache.kafka.clients.ClientDnsLookup;
import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.MetadataRecoveryStrategy;
+import org.apache.kafka.clients.NetworkClient;
import org.apache.kafka.clients.consumer.CommitFailedException;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.RetriableCommitFailedException;
+import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler;
+import org.apache.kafka.clients.consumer.internals.metrics.AsyncConsumerMetrics;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.BootstrapResolutionException;
import org.apache.kafka.common.errors.GroupAuthorizationException;
import org.apache.kafka.common.errors.InvalidCommitOffsetSizeException;
import org.apache.kafka.common.errors.OffsetMetadataTooLarge;
@@ -33,6 +41,7 @@ import org.apache.kafka.common.errors.StaleMemberEpochException;
import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.errors.TopicAuthorizationException;
import org.apache.kafka.common.errors.UnknownMemberIdException;
+import org.apache.kafka.common.internals.ClusterResourceListeners;
import org.apache.kafka.common.message.OffsetCommitRequestData;
import org.apache.kafka.common.message.OffsetCommitResponseData;
import org.apache.kafka.common.message.OffsetFetchRequestData;
@@ -50,6 +59,7 @@ import org.apache.kafka.common.requests.RequestHeader;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.common.utils.internals.LogContext;
+import org.apache.kafka.test.MockSelector;
import org.apache.kafka.test.TestUtils;
import org.junit.jupiter.api.BeforeEach;
@@ -731,6 +741,83 @@ public class CommitRequestManagerTest {
assertPoll(1, commitRequestManager);
}
+ @Test
+ public void testMaximumTimeToWaitWhenCoordinatorUnknownDoesNotSpin() {
+ CommitRequestManager commitRequestManager = create(true, 100);
+ when(coordinatorRequestManager.coordinator()).thenReturn(Optional.empty());
+
+ time.sleep(100);
+ long result = commitRequestManager.maximumTimeToWait(time.milliseconds());
+
+ assertTrue(result > 0,
+ "maximumTimeToWait must be > 0 when the coordinator is unknown to avoid a busy-spin; got " + result);
+ assertEquals(100, result);
+ }
+
+ @Test
+ public void testMaximumTimeToWaitDoesNotSpinDuringRealBootstrapDnsResolution() throws Exception {
+ long bootstrapResolveTimeoutMs = 1000;
+
+ BootstrapConfiguration bootstrapConfiguration = BootstrapConfiguration.enabled(
+ List.of("unresolvable.invalid:9092"),
+ ClientDnsLookup.USE_ALL_DNS_IPS,
+ bootstrapResolveTimeoutMs,
+ retryBackoffMs
+ );
+
+ ConsumerConfig config = new ConsumerConfig(Map.of(
+ ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class,
+ ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class,
+ ConsumerConfig.GROUP_ID_CONFIG, DEFAULT_GROUP_ID,
+ ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "unresolvable.invalid:9092",
+ // Much shorter than bootstrapResolveTimeoutMs, so the auto-commit timer expires several
+ // times while the coordinator is still (and will remain, since DNS never resolves) unknown.
+ ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "100",
+ ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"
+ ));
+
+ ConsumerMetadata metadata = new ConsumerMetadata(config, subscriptionState, logContext, new ClusterResourceListeners());
+
+ MockSelector selector = new MockSelector(time);
+ NetworkClient networkClient = new NetworkClient(selector, metadata, "test-client",
+ Integer.MAX_VALUE, 50, 1000, 64 * 1024, 64 * 1024, 1000, 5000, 30000,
+ time, false, new ApiVersions(), logContext,
+ MetadataRecoveryStrategy.NONE, bootstrapConfiguration, false);
+
+
+ CoordinatorRequestManager realCoordinatorRequestManager = new CoordinatorRequestManager(
+ logContext, retryBackoffMs, retryBackoffMaxMs, DEFAULT_GROUP_ID);
+
+ CommitRequestManager realCommitRequestManager = new CommitRequestManager(
+ time, logContext, subscriptionState, config, realCoordinatorRequestManager,
+ mock(OffsetCommitCallbackInvoker.class), DEFAULT_GROUP_ID, Optional.empty(), retryBackoffMs,
+ retryBackoffMaxMs, OptionalDouble.of(0), new Metrics(), metadata);
+
+ try (NetworkClientDelegate networkClientDelegate = new NetworkClientDelegate(time, config, logContext, networkClient, metadata,
+ mock(BackgroundEventHandler.class), false, mock(AsyncConsumerMetrics.class));
+ ) {
+ long deadline = time.milliseconds() + bootstrapResolveTimeoutMs + 3000;
+ boolean sawBootstrapException = false;
+
+ while (time.milliseconds() < deadline) {
+ // Drives the real NetworkClient's ensureBootstrapped()/async DNS resolution forward;
+ // the coordinator never becomes known since there is no real broker to respond.
+ networkClientDelegate.poll(50, time.milliseconds());
+
+ long waitMs = realCommitRequestManager.maximumTimeToWait(time.milliseconds());
+ assertTrue(waitMs > 0, "maximumTimeToWait must be > 0 while real bootstrap DNS resolution is pending; got " + waitMs);
+
+ Optional<Exception> metadataError = networkClientDelegate.getAndClearMetadataError();
+ if (metadataError.isPresent()) {
+ assertInstanceOf(BootstrapResolutionException.class, metadataError.get());
+ sawBootstrapException = true;
+ break;
+ }
+ }
+ assertTrue(sawBootstrapException, "Expected a real BootstrapResolutionException within " + (bootstrapResolveTimeoutMs + 3000) + "ms");
+ }
+ }
+
@Test
public void testOffsetFetchRequestEnsureDuplicatedRequestSucceed() {
CommitRequestManager commitRequestManager = create(true, 100);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManagerTest.java
index 737c436b4cc..28f4df825ac 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManagerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManagerTest.java
@@ -2512,6 +2512,7 @@ class StreamsGroupHeartbeatRequestManagerTest {
) {
final StreamsGroupHeartbeatRequestManager heartbeatRequestManager = createStreamsGroupHeartbeatRequestManager();
final Timer pollTimer = timerMockedConstruction.constructed().get(0);
+ when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(coordinatorNode));
when(membershipManager.shouldNotWaitForHeartbeatInterval()).thenReturn(true);
time.sleep(1234);
@@ -2540,6 +2541,7 @@ class StreamsGroupHeartbeatRequestManagerTest {
) {
final StreamsGroupHeartbeatRequestManager heartbeatRequestManager = createStreamsGroupHeartbeatRequestManager();
final Timer pollTimer = timerMockedConstruction.constructed().get(0);
+ when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(coordinatorNode));
when(membershipManager.shouldNotWaitForHeartbeatInterval()).thenReturn(shouldNotWaitForHeartbeatInterval);
time.sleep(1234);
@@ -2563,6 +2565,7 @@ class StreamsGroupHeartbeatRequestManagerTest {
) {
final StreamsGroupHeartbeatRequestManager heartbeatRequestManager = createStreamsGroupHeartbeatRequestManager();
final Timer pollTimer = timerMockedConstruction.constructed().get(0);
+ when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(coordinatorNode));
time.sleep(1234);
final long maximumTimeToWait = heartbeatRequestManager.maximumTimeToWait(time.milliseconds());
@@ -2572,6 +2575,24 @@ class StreamsGroupHeartbeatRequestManagerTest {
}
}
+ @Test
+ public void testMaximumTimeToWaitWhenCoordinatorUnknownDoesNotSpin() {
+ try (
+ final MockedConstruction<Timer> timerMockedConstruction = mockConstruction(Timer.class);
+ final MockedConstruction<HeartbeatRequestState> heartbeatRequestStateMockedConstruction = mockConstruction(
+ HeartbeatRequestState.class,
+ (mock, context) -> when(mock.heartbeatIntervalMs()).thenReturn(6000L))
+ ) {
+ final StreamsGroupHeartbeatRequestManager heartbeatRequestManager = createStreamsGroupHeartbeatRequestManager();
+ when(coordinatorRequestManager.coordinator()).thenReturn(Optional.empty());
+ time.sleep(1234);
+
+ final long maximumTimeToWait = heartbeatRequestManager.maximumTimeToWait(time.milliseconds());
+
+ assertEquals(6000L, maximumTimeToWait);
+ }
+ }
+
@Test
public void testResetPollTimer() {
try (final MockedConstruction<Timer> pollTimerMockedConstruction = mockConstruction(Timer.class)) {