You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by rf...@apache.org on 2026/06/24 12:20:36 UTC
(pulsar-connectors) 01/01: fix(kinesis): prevent connector crash on checkpoint failure
This is an automated email from the ASF dual-hosted git repository.
freeznet pushed a commit to branch freeznet/kinesis-exception
in repository https://gitbox.apache.org/repos/asf/pulsar-connectors.git
commit fd89dfd314d9c705be05e839cab28161f53a0d60
Author: Rui Fu <rf...@apache.org>
AuthorDate: Wed Jun 24 20:20:20 2026 +0800
fix(kinesis): prevent connector crash on checkpoint failure
---
.../pulsar/io/kinesis/KinesisRecordProcessor.java | 53 +++++++++++++++-------
.../io/kinesis/KinesisRecordProcessorTest.java | 45 ++++++++++++++++++
2 files changed, 82 insertions(+), 16 deletions(-)
diff --git a/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java b/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
index daee5041..1a6f2d42 100644
--- a/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
+++ b/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessor.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.io.kinesis;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -78,23 +79,40 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
checkpointer.checkpoint(checkpoint.sequenceNumber(), checkpoint.subSequenceNumber());
lastCheckpointSequenceNumber = checkpoint;
log.info("Successfully checkpointed shard {} at {}", kinesisShardId, checkpoint);
- isCheckpointing.set(false);
- checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+ scheduleNextCheckpoint();
} catch (ThrottlingException | KinesisClientLibDependencyException e) {
if (attempt >= numRetries) {
- log.error("Checkpoint for shard {} failed after {} attempts at {}. Terminating.",
- kinesisShardId, numRetries, checkpoint, e);
- sourceContext.fatal(e);
+ // A failed periodic checkpoint is recoverable: skip this round instead of terminating the
+ // connector, and let the next scheduled checkpoint retry with the latest acked position.
+ log.error("Checkpoint for shard {} failed after {} attempts at {}. Skipping this round; "
+ + "will retry on the next interval.", kinesisShardId, numRetries, checkpoint, e);
+ scheduleNextCheckpoint();
} else {
log.warn("Throttling/Dependency error on checkpoint for shard {} at {}. Scheduling retry {} "
+ "after {}ms.", kinesisShardId, checkpoint, attempt + 1, backoffTime);
- checkpointExecutor.schedule(() -> tryCheckpointWithRetry(checkpointer, checkpoint, attempt + 1),
- backoffTime, TimeUnit.MILLISECONDS);
+ safeSchedule(() -> tryCheckpointWithRetry(checkpointer, checkpoint, attempt + 1), backoffTime);
}
} catch (Exception e) {
- log.error("Caught a non-retryable exception for shard {} during checkpoint at {}. Terminating.",
- kinesisShardId, checkpoint, e);
- sourceContext.fatal(e);
+ // Non-retryable errors (e.g. KCL's IllegalArgumentException when an out-of-order ack tries to
+ // checkpoint a position behind the last checkpoint) must NOT crash the connector. Skip this
+ // round; a later ack with a higher position will advance the checkpoint on a subsequent cycle.
+ log.error("Non-retryable exception during checkpoint for shard {} at {}. Skipping this round; "
+ + "will retry on the next interval.", kinesisShardId, checkpoint, e);
+ scheduleNextCheckpoint();
+ }
+ }
+
+ private void scheduleNextCheckpoint() {
+ isCheckpointing.set(false);
+ safeSchedule(this::triggerCheckpoint, checkpointInterval);
+ }
+
+ private void safeSchedule(Runnable task, long delayMillis) {
+ try {
+ checkpointExecutor.schedule(task, delayMillis, TimeUnit.MILLISECONDS);
+ } catch (RejectedExecutionException e) {
+ // The checkpoint executor has been shut down (connector is closing); nothing left to schedule.
+ log.debug("Checkpoint executor rejected a task for shard {} (likely shutting down).", kinesisShardId);
}
}
@@ -116,7 +134,7 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
log.info("Initializing KinesisRecordProcessor for shard {}, extendedSequenceNumber: {}, pendingCheckSeq: {}",
kinesisShardId, initializationInput.extendedSequenceNumber(),
initializationInput.pendingCheckpointSequenceNumber());
- checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+ safeSchedule(this::triggerCheckpoint, checkpointInterval);
}
private void triggerCheckpoint() {
@@ -128,13 +146,15 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
lastCheckpointSequenceNumber)) {
tryCheckpointWithRetry(checkpointer, currentCheckpoint, 1);
} else {
- isCheckpointing.set(false);
- checkpointExecutor.schedule(this::triggerCheckpoint, checkpointInterval, TimeUnit.MILLISECONDS);
+ scheduleNextCheckpoint();
}
}
} catch (Throwable e) {
- log.error("Error while triggering checkpoint for shard {}. Terminating.", kinesisShardId, e);
- sourceContext.fatal(e);
+ // Defensive: never let a checkpoint-trigger failure crash the connector. Keep the loop alive
+ // so the next scheduled cycle can make progress.
+ log.error("Unexpected error while triggering checkpoint for shard {}. Skipping this round.",
+ kinesisShardId, e);
+ scheduleNextCheckpoint();
}
}
@@ -203,8 +223,9 @@ public class KinesisRecordProcessor implements ShardRecordProcessor {
}
}
} catch (Exception e) {
+ // Defensive: a failed final checkpoint must not crash the connector. Under at-least-once
+ // semantics the un-checkpointed records will simply be reprocessed by the next lease owner.
log.error("Failed to perform final checkpoint for shard {}. Data may be reprocessed.", kinesisShardId, e);
- sourceContext.fatal(e);
}
}
diff --git a/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java b/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
index 70c04fa8..4e8e7342 100644
--- a/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
+++ b/kinesis/src/test/java/org/apache/pulsar/io/kinesis/KinesisRecordProcessorTest.java
@@ -41,6 +41,7 @@ import software.amazon.kinesis.lifecycle.events.InitializationInput;
import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
import software.amazon.kinesis.lifecycle.events.ShardEndedInput;
import software.amazon.kinesis.lifecycle.events.ShutdownRequestedInput;
+import software.amazon.kinesis.exceptions.ThrottlingException;
import software.amazon.kinesis.processor.RecordProcessorCheckpointer;
import software.amazon.kinesis.retrieval.KinesisClientRecord;
@@ -121,6 +122,50 @@ public class KinesisRecordProcessorTest {
verify(checkpointer, times(1)).checkpoint("seq-1", 7L);
}
+ @Test
+ public void testNonRetryableCheckpointFailureDoesNotCrashConnector() throws Exception {
+ // Arrange: initialize schedules the first checkpoint task.
+ recordProcessor.initialize(createMockInitializationInput());
+ verify(checkpointExecutor, times(1))
+ .schedule(scheduledTaskCaptor.capture(), anyLong(), any(TimeUnit.class));
+ Runnable scheduledCheckpointTask = scheduledTaskCaptor.getValue();
+ recordProcessor.processRecords(createMockProcessRecordsInput(checkpointer));
+
+ // Simulate KCL rejecting a backward / out-of-range checkpoint (an IllegalArgumentException).
+ Mockito.doThrow(new IllegalArgumentException("Could not checkpoint ... did not fall into acceptable range"))
+ .when(checkpointer).checkpoint(any(String.class), anyLong());
+ recordProcessor.updateSequenceNumberToCheckpoint("seq-1", 5L);
+
+ // Act
+ scheduledCheckpointTask.run();
+
+ // Assert: the connector must not be terminated, and the checkpoint loop must be rescheduled
+ // (one schedule from initialize + one from the failed round).
+ verify(sourceContext, never()).fatal(any());
+ verify(checkpointExecutor, times(2)).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+ }
+
+ @Test
+ public void testExhaustedRetriableCheckpointFailureDoesNotCrashConnector() throws Exception {
+ // Arrange: numRetries == 1 (see setup), so the first throttling failure already exhausts retries.
+ recordProcessor.initialize(createMockInitializationInput());
+ verify(checkpointExecutor, times(1))
+ .schedule(scheduledTaskCaptor.capture(), anyLong(), any(TimeUnit.class));
+ Runnable scheduledCheckpointTask = scheduledTaskCaptor.getValue();
+ recordProcessor.processRecords(createMockProcessRecordsInput(checkpointer));
+
+ Mockito.doThrow(new ThrottlingException("throttled"))
+ .when(checkpointer).checkpoint(any(String.class), anyLong());
+ recordProcessor.updateSequenceNumberToCheckpoint("seq-1", 1L);
+
+ // Act
+ scheduledCheckpointTask.run();
+
+ // Assert: retries exhausted, but the connector survives and the loop is rescheduled.
+ verify(sourceContext, never()).fatal(any());
+ verify(checkpointExecutor, times(2)).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
+ }
+
@Test(timeOut = 3000)
public void testShardEndedTimeoutPathPerformsBestEffortCheckpoint() throws Exception {
// Arrange: Process two records, but only ack one.