You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by mj...@apache.org on 2024/06/30 18:52:42 UTC

(kafka) branch trunk updated: KAFKA-16508: Streams custom handler should handle the timeout exceptions (#16450)

This is an automated email from the ASF dual-hosted git repository.

mjsax pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 15a4501bded KAFKA-16508: Streams custom handler should handle the timeout exceptions (#16450)
15a4501bded is described below

commit 15a4501bded513822485dd85fa6258e16f1571ca
Author: Alieh Saeedi <10...@users.noreply.github.com>
AuthorDate: Sun Jun 30 20:52:36 2024 +0200

    KAFKA-16508: Streams custom handler should handle the timeout exceptions (#16450)
    
    For a non-existing output topic, Kafka Streams ends up in an infinite retry loop, because the returned TimeoutException extends RetriableException.
    
    This PR updates the error handling pass for this case and instead of retrying calls the ProductionExceptionHandler to allow breaking the infinite retry loop.
    
    Reviewers: Matthias J. Sax <ma...@confluent.io>
---
 .../processor/internals/RecordCollectorImpl.java   |  13 +-
 .../integration/CustomHandlerIntegrationTest.java  | 166 +++++++++++++++++++++
 .../processor/internals/RecordCollectorTest.java   |  45 ++++++
 3 files changed, 223 insertions(+), 1 deletion(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
index 597d36e5d7b..16d67666ccb 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RecordCollectorImpl.java
@@ -34,6 +34,7 @@ import org.apache.kafka.common.errors.SecurityDisabledException;
 import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.errors.TimeoutException;
 import org.apache.kafka.common.errors.UnknownServerException;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.serialization.Serializer;
@@ -306,7 +307,7 @@ public class RecordCollectorImpl implements RecordCollector {
                 "indicating the task may be migrated out";
             sendException.set(new TaskMigratedException(errorMessage, exception));
         } else {
-            if (exception instanceof RetriableException) {
+            if (isRetriable(exception)) {
                 errorMessage += "\nThe broker is either slow or in bad state (like not having enough replicas) in responding the request, " +
                     "or the connection to broker was interrupted sending the request or receiving the response. " +
                     "\nConsider overwriting `max.block.ms` and /or " +
@@ -326,6 +327,16 @@ public class RecordCollectorImpl implements RecordCollector {
         log.error(errorMessage, exception);
     }
 
+    /**
+     * The `TimeoutException` with root cause `UnknownTopicOrPartitionException` is considered as non-retriable
+     * (despite `TimeoutException` being a subclass of `RetriableException`, this particular case is explicitly excluded).
+    */
+    private boolean isRetriable(final Exception exception) {
+        return exception instanceof RetriableException &&
+                (!(exception instanceof TimeoutException) || exception.getCause() == null
+                        || !(exception.getCause() instanceof UnknownTopicOrPartitionException));
+    }
+
     private boolean isFatalException(final Exception exception) {
         final boolean securityException = exception instanceof AuthenticationException ||
             exception instanceof AuthorizationException ||
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/CustomHandlerIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/CustomHandlerIntegrationTest.java
new file mode 100644
index 00000000000..3eea2ec7d84
--- /dev/null
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/CustomHandlerIntegrationTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.integration;
+
+import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
+import org.apache.kafka.common.serialization.IntegerSerializer;
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.KafkaStreams.State;
+import org.apache.kafka.streams.KeyValue;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.errors.StreamsException;
+import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler;
+import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
+import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
+import org.apache.kafka.streams.kstream.Consumed;
+import org.apache.kafka.streams.kstream.Produced;
+import org.apache.kafka.test.TestUtils;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.Timeout;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.safeUniqueTestName;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+
+@Timeout(600)
+@Tag("integration")
+public class CustomHandlerIntegrationTest {
+    private static final int NUM_BROKERS = 1;
+    private static final int NUM_THREADS = 2;
+    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(NUM_BROKERS,
+            Utils.mkProperties(Collections.singletonMap("auto.create.topics.enable", "false")));
+
+    @BeforeAll
+    public static void startCluster() throws IOException {
+        CLUSTER.start();
+    }
+
+    @AfterAll
+    public static void closeCluster() {
+        CLUSTER.stop();
+    }
+
+    private final long timeout = 60000;
+
+    // topic name
+    private static final String STREAM_INPUT = "STREAM_INPUT";
+    private static final String NON_EXISTING_TOPIC = "non_existing_topic";
+
+    private KafkaStreams kafkaStreams;
+    AtomicReference<Throwable> caughtException;
+    Topology topology;
+    private String appId;
+
+    @BeforeEach
+    public void before(final TestInfo testInfo) throws InterruptedException {
+        final StreamsBuilder builder = new StreamsBuilder();
+        CLUSTER.createTopics(STREAM_INPUT);
+        caughtException = new AtomicReference<>();
+        final String safeTestName = safeUniqueTestName(testInfo);
+        appId = "app-" + safeTestName;
+
+
+        builder.stream(STREAM_INPUT, Consumed.with(Serdes.Integer(), Serdes.String()))
+                .to(NON_EXISTING_TOPIC, Produced.with(Serdes.Integer(), Serdes.String()));
+        produceRecords();
+        topology = builder.build();
+    }
+
+    @AfterEach
+    public void after() throws InterruptedException {
+        CLUSTER.deleteTopics(STREAM_INPUT);
+        if (kafkaStreams != null) {
+            kafkaStreams.close();
+            kafkaStreams.cleanUp();
+        }
+    }
+
+    private void produceRecords() {
+        final Properties props = TestUtils.producerConfig(
+                CLUSTER.bootstrapServers(),
+                IntegerSerializer.class,
+                StringSerializer.class,
+                new Properties());
+        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
+                STREAM_INPUT,
+                Collections.singletonList(new KeyValue<>(1, "A")),
+                props,
+                CLUSTER.time.milliseconds() + 2
+        );
+    }
+
+    private Properties getCommonProperties() {
+        final Properties streamsConfiguration = new Properties();
+        streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
+        streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
+        streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.IntegerSerde.class);
+        streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
+        streamsConfiguration.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, NUM_THREADS);
+        return streamsConfiguration;
+    }
+
+    private void closeApplication(final Properties streamsConfiguration) throws Exception {
+        kafkaStreams.close();
+        kafkaStreams.cleanUp();
+        IntegrationTestUtils.purgeLocalStreamsState(streamsConfiguration);
+    }
+
+    @Test
+    public void shouldThrowStreamsExceptionWithMissingTopicAndDefaultExceptionHandler() throws Exception {
+        final Properties streamsConfiguration = getCommonProperties();
+        kafkaStreams = new KafkaStreams(topology, streamsConfiguration);
+        kafkaStreams.setUncaughtExceptionHandler(e -> {
+            caughtException.set(e);
+            return StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT;
+        });
+        kafkaStreams.start();
+        TestUtils.waitForCondition(
+                () -> kafkaStreams.state() == State.RUNNING,
+                timeout,
+                () -> "Kafka Streams application did not reach state RUNNING in " + timeout + " ms");
+        while (true) {
+            if (caughtException.get() != null) {
+                final Throwable throwable = caughtException.get();
+                assertInstanceOf(StreamsException.class, throwable);
+                assertInstanceOf(TimeoutException.class, throwable.getCause());
+                assertInstanceOf(UnknownTopicOrPartitionException.class, throwable.getCause().getCause());
+                closeApplication(streamsConfiguration);
+                break;
+            } else {
+                Thread.sleep(100);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
index f57d14a6a0c..5222759bc5e 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/RecordCollectorTest.java
@@ -34,6 +34,7 @@ import org.apache.kafka.common.errors.InvalidProducerEpochException;
 import org.apache.kafka.common.errors.ProducerFencedException;
 import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.errors.UnknownTopicOrPartitionException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeader;
@@ -92,6 +93,7 @@ import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetric
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.IsEqual.equalTo;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -1351,6 +1353,49 @@ public class RecordCollectorTest {
         collector.closeClean();
     }
 
+    @Test
+    public void shouldThrowStreamsExceptionOnUnknownTopicOrPartitionExceptionWithDefaultExceptionHandler() {
+        final KafkaException exception = new TimeoutException("KABOOM!", new UnknownTopicOrPartitionException());
+        final RecordCollector collector = new RecordCollectorImpl(
+                logContext,
+                taskId,
+                getExceptionalStreamsProducerOnSend(exception),
+                productionExceptionHandler,
+                streamsMetrics,
+                topology
+        );
+
+        collector.send(topic, "3", "0", null, null, stringSerializer, stringSerializer, null, null, streamPartitioner);
+
+        // With default handler which returns FAIL, flush() throws StreamsException with TimeoutException cause,
+        // otherwise it would throw a TaskCorruptedException with null cause
+        final StreamsException thrown = assertThrows(StreamsException.class, collector::flush);
+        assertEquals(exception, thrown.getCause());
+        assertThat(
+                thrown.getMessage(),
+                equalTo("Error encountered sending record to topic topic for task 0_0 due to:" +
+                        "\norg.apache.kafka.common.errors.TimeoutException: KABOOM!" +
+                        "\nException handler choose to FAIL the processing, no more records would be sent.")
+        );
+    }
+
+    @Test
+    public void shouldNotThrowTaskCorruptedExceptionOnUnknownTopicOrPartitionExceptionUsingAlwaysContinueExceptionHandler() {
+        final KafkaException exception = new TimeoutException("KABOOM!", new UnknownTopicOrPartitionException());
+        final RecordCollector collector = new RecordCollectorImpl(
+                logContext,
+                taskId,
+                getExceptionalStreamsProducerOnSend(exception),
+                new AlwaysContinueProductionExceptionHandler(),
+                streamsMetrics,
+                topology
+        );
+
+        collector.send(topic, "3", "0", null, null, stringSerializer, stringSerializer, null, null, streamPartitioner);
+
+        assertDoesNotThrow(collector::flush);
+    }
+
     @Test
     public void shouldNotAbortTxnOnEOSCloseDirtyIfNothingSent() {
         final AtomicBoolean functionCalled = new AtomicBoolean(false);