You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by rf...@apache.org on 2026/06/24 12:20:36 UTC

(pulsar-connectors) 01/01: fix(kinesis): prevent connector crash on checkpoint failure

This is an automated email from the ASF dual-hosted git repository.

freeznet pushed a commit to branch freeznet/kinesis-exception
in repository https://gitbox.apache.org/repos/asf/pulsar-connectors.git

commit fd89dfd314d9c705be05e839cab28161f53a0d60
Author: Rui Fu <rf...@apache.org>
AuthorDate: Wed Jun 24 20:20:20 2026 +0800

    fix(kinesis): prevent connector crash on checkpoint failure
---
 .../pulsar/io/kinesis/KinesisRecordProcessor.java  | 53 +++++++++++++++-------
 .../io/kinesis/KinesisRecordProcessorTest.java     | 45 ++++++++++++++++++
 2 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java b/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
index daee5041..1a6f2d42 100644
--- a/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
+++ b/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.io.kinesis;
 
 import java.util.Set;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -78,23 +79,40 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
             checkpointer.checkpoint(checkpoint.sequenceNumber(), checkpoint.subSequenceNumber());
             lastCheckpointSequenceNumber = checkpoint;
             log.info("Successfully checkpointed shard {} at {}", kinesisShardId, checkpoint);
-            isCheckpointing.set(false);
-            checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+            scheduleNextCheckpoint();
         } catch (ThrottlingException | KinesisClientLibDependencyException e) {
             if (attempt >= numRetries) {
-                log.error("Checkpoint for shard {} failed after {} attempts at {}. Terminating.",
-                        kinesisShardId, numRetries, checkpoint, e);
-                sourceContext.fatal(e);
+                // A failed periodic checkpoint is recoverable: skip this round instead of terminating the
+                // connector, and let the next scheduled checkpoint retry with the latest acked position.
+                log.error("Checkpoint for shard {} failed after {} attempts at {}. Skipping this round; "
+                        + "will retry on the next interval.", kinesisShardId, numRetries, checkpoint, e);
+                scheduleNextCheckpoint();
             } else {
                 log.warn("Throttling/Dependency error on checkpoint for shard {} at {}. Scheduling retry {} "
                                 + "after {}ms.", kinesisShardId, checkpoint, attempt + 1, backoffTime);
-                checkpointExecutor.schedule(() -> tryCheckpointWithRetry(checkpointer, checkpoint, attempt + 1),
-                        backoffTime, TimeUnit.MILLISECONDS);
+                safeSchedule(() -> tryCheckpointWithRetry(checkpointer, checkpoint, attempt + 1), backoffTime);
             }
         } catch (Exception e) {
-            log.error("Caught a non-retryable exception for shard {} during checkpoint at {}. Terminating.",
-                    kinesisShardId, checkpoint, e);
-            sourceContext.fatal(e);
+            // Non-retryable errors (e.g. KCL's IllegalArgumentException when an out-of-order ack tries to
+            // checkpoint a position behind the last checkpoint) must NOT crash the connector. Skip this
+            // round; a later ack with a higher position will advance the checkpoint on a subsequent cycle.
+            log.error("Non-retryable exception during checkpoint for shard {} at {}. Skipping this round; "
+                    + "will retry on the next interval.", kinesisShardId, checkpoint, e);
+            scheduleNextCheckpoint();
+        }
+    }
+
+    private void scheduleNextCheckpoint() {
+        isCheckpointing.set(false);
+        safeSchedule(this::triggerCheckpoint, checkpointInterval);
+    }
+
+    private void safeSchedule(Runnable task, long delayMillis) {
+        try {
+            checkpointExecutor.schedule(task, delayMillis, TimeUnit.MILLISECONDS);
+        } catch (RejectedExecutionException e) {
+            // The checkpoint executor has been shut down (connector is closing); nothing left to schedule.
+            log.debug("Checkpoint executor rejected a task for shard {} (likely shutting down).", kinesisShardId);
         }
     }
 
@@ -116,7 +134,7 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
         log.info("Initializing KinesisRecordProcessor for shard {}, extendedSequenceNumber: {}, pendingCheckSeq: {}",
                 kinesisShardId, initializationInput.extendedSequenceNumber(),
                 initializationInput.pendingCheckpointSequenceNumber());
-        checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+        safeSchedule(this::triggerCheckpoint, checkpointInterval);
     }
 
     private void triggerCheckpoint() {
@@ -128,13 +146,15 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
                         lastCheckpointSequenceNumber)) {
                     tryCheckpointWithRetry(checkpointer, currentCheckpoint, 1);
                 } else {
-                    isCheckpointing.set(false);
-                    checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+                    scheduleNextCheckpoint();
                 }
             }
         } catch (Throwable e) {
-            log.error("Error while triggering checkpoint for shard {}. Terminating.", kinesisShardId, e);
-            sourceContext.fatal(e);
+            // Defensive: never let a checkpoint-trigger failure crash the connector. Keep the loop alive
+            // so the next scheduled cycle can make progress.
+            log.error("Unexpected error while triggering checkpoint for shard {}. Skipping this round.",
+                    kinesisShardId, e);
+            scheduleNextCheckpoint();
         }
     }
 
@@ -203,8 +223,9 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
                 }
             }
         } catch (Exception e) {
+            // Defensive: a failed final checkpoint must not crash the connector. Under at-least-once
+            // semantics the un-checkpointed records will simply be reprocessed by the next lease owner.
             log.error("Failed to perform final checkpoint for shard {}. Data may be reprocessed.", kinesisShardId, e);
-            sourceContext.fatal(e);
         }
     }
 
diff --git a/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java b/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
index 70c04fa8..4e8e7342 100644
--- a/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
+++ b/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
@@ -41,6 +41,7 @@ import software.amazon.kinesis.lifecycle.events.InitializationInput;
 import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
 import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
 import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput;
+import software.amazon.kinesis.exceptions.ThrottlingException;
 import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
 import software.amazon.kinesis.retrieval.KinesisClientRecord;
 
@@ -121,6 +122,50 @@ public class KinesisRecordProcessorTest {
         verify(checkpointer, times(1)).checkpoint("seq-1", 7L);
     }
 
+    @Test
+    public void testNonRetryableCheckpointFailureDoesNotCrashConnector() throws Exception {
+        // Arrange: initialize schedules the first checkpoint task.
+        recordProcessor.initialize(createMockInitializationInput());
+        verify(checkpointExecutor, times(1))
+                .schedule(scheduledTaskCaptor.capture(), anyLong(), any(TimeUnit.class));
+        Runnable scheduledCheckpointTask = scheduledTaskCaptor.getValue();
+        recordProcessor.processRecords(createMockProcessRecordsInput(checkpointer));
+
+        // Simulate KCL rejecting a backward / out-of-range checkpoint (an IllegalArgumentException).
+        Mockito.doThrow(new IllegalArgumentException("Could not checkpoint ... did not fall into acceptable range"))
+                .when(checkpointer).checkpoint(any(String.class), anyLong());
+        recordProcessor.updateSequenceNumberToCheckpoint("seq-1", 5L);
+
+        // Act
+        scheduledCheckpointTask.run();
+
+        // Assert: the connector must not be terminated, and the checkpoint loop must be rescheduled
+        // (one schedule from initialize + one from the failed round).
+        verify(sourceContext, never()).fatal(any());
+        verify(checkpointExecutor, times(2)).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+    }
+
+    @Test
+    public void testExhaustedRetriableCheckpointFailureDoesNotCrashConnector() throws Exception {
+        // Arrange: numRetries == 1 (see setup), so the first throttling failure already exhausts retries.
+        recordProcessor.initialize(createMockInitializationInput());
+        verify(checkpointExecutor, times(1))
+                .schedule(scheduledTaskCaptor.capture(), anyLong(), any(TimeUnit.class));
+        Runnable scheduledCheckpointTask = scheduledTaskCaptor.getValue();
+        recordProcessor.processRecords(createMockProcessRecordsInput(checkpointer));
+
+        Mockito.doThrow(new ThrottlingException("throttled"))
+                .when(checkpointer).checkpoint(any(String.class), anyLong());
+        recordProcessor.updateSequenceNumberToCheckpoint("seq-1", 1L);
+
+        // Act
+        scheduledCheckpointTask.run();
+
+        // Assert: retries exhausted, but the connector survives and the loop is rescheduled.
+        verify(sourceContext, never()).fatal(any());
+        verify(checkpointExecutor, times(2)).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+    }
+
     @Test(timeOut = 3000)
     public void testShardEndedTimeoutPathPerformsBestEffortCheckpoint() throws Exception {
         // Arrange: Process two records, but only ack one.