You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by bb...@apache.org on 2026/09/08 17:27:05 UTC
(kafka) 03/04: KAFKA-20876: Windowed restore optimisation gives up when `endOffset-1` is a transaction control record (#23086)
This is an automated email from the ASF dual-hosted git repository.
bbejeck pushed a commit to branch 4.3
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit b3add3983546158bd2eeaaf2fa62e8f1b68edb89
Author: Alan Lau <72...@users.noreply.github.com>
AuthorDate: Thu Aug 13 16:11:06 2026 -0400
KAFKA-20876: Windowed restore optimisation gives up when `endOffset-1` is a transaction control record (#23086)
Jira: https://issues.apache.org/jira/browse/KAFKA-20876
When restoring a windowed store with no checkpoint,
`StoreChangelogReader` seeks to `latestTimestamp - retentionPeriod`
rather than log-start, since data older than the retention is discarded
on write. It learns `latestTimestamp` by seeking to `endOffset - 1` and
polling once in `seekNewPartitions`; an empty poll falls back to
`seekToBeginning`.
The probe now reads a window back from the end rather than a single
offset, widens it when that window holds no record, and keeps polling a
window for as long as it is still resolving partitions.
Reviewers: Bill Bejeck <bb...@apache.org>
---
.../processor/internals/StoreChangelogReader.java | 168 +++++-
.../internals/StoreChangelogReaderTest.java | 592 +++++++++++++++++++++
2 files changed, 748 insertions(+), 12 deletions(-)
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index e634aae279b..3c82dcc7259 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -55,6 +55,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
@@ -205,8 +206,34 @@ public class StoreChangelogReader implements ChangelogReader {
private static final long DEFAULT_OFFSET_UPDATE_MS = Duration.ofMinutes(5L).toMillis();
+ // Windows the probe reads back from each partition's end. Not 1: under EOS the last offset is
+ // usually a transaction control record, never delivered to a consumer, so probing there is a
+ // guaranteed empty poll.
+ private static final long[] PROBE_WINDOWS = {128L, 512L, 2048L};
+
+ // One empty poll proves nothing: poll() returns as soon as any fetch lands, so a partition can
+ // come back empty because another was served first.
+ private static final int PROBE_IDLE_POLLS = 3;
+
+ // Floor on the waiting a window is owed before it may be called empty: polls that return
+ // immediately have not waited on a fetch, and the floor has to cover a widening, whose re-seek
+ // discards the in-flight fetch and queues behind it -- one fetch in flight per broker.
+ private static final int PROBE_MIN_WAIT_POLLS = 10;
+
+ // Ceiling per window, so polls that return without waiting cannot spin the probe.
+ private static final int PROBE_MAX_POLLS = 30;
+
+ // How long a partition whose probe fell back is left alone, so a task corrupted, wiped and
+ // re-registered in a loop cannot probe on every iteration. Only a failed probe arms it, so a
+ // probe that is working is never suppressed.
+ private static final Duration PROBE_RETRY_BACKOFF = Duration.ofSeconds(60);
+
private ChangelogReaderState state;
+ // deliberately outlives unregister: a changelog that is corrupted and re-registered in a loop
+ // must not forget that probing it has just failed
+ private final Map<TopicPartition, Long> probeFailedAtMs = new HashMap<>();
+
private final Time time;
private final Logger log;
private final Duration pollTime;
@@ -1019,7 +1046,13 @@ public class StoreChangelogReader implements ChangelogReader {
} else {
final long retentionPeriod = storeMetadata.retentionPeriod();
if (retentionPeriod > 0 && retentionPeriod != Long.MAX_VALUE) {
- newWindowedPartitionsRetention.put(partition, retentionPeriod);
+ if (probeIsDue(partition)) {
+ newWindowedPartitionsRetention.put(partition, retentionPeriod);
+ } else {
+ log.debug("Start restoring changelog partition {} from the beginning offset to end offset {} " +
+ "since its last attempt to seek past expired data failed.", partition, recordEndOffset(endOffset));
+ newSeekToBeginningPartitions.add(partition);
+ }
} else {
final StateStore store = storeMetadata.store();
if (store instanceof WindowStore || store instanceof SessionStore) {
@@ -1096,21 +1129,25 @@ public class StoreChangelogReader implements ChangelogReader {
final Map<TopicPartition, Long> endOffsets =
restoreConsumer.endOffsets(windowedPartitionsRetention.keySet());
+ final Map<TopicPartition, Long> beginningOffsets =
+ restoreConsumer.beginningOffsets(windowedPartitionsRetention.keySet());
for (final TopicPartition partition : windowedPartitionsRetention.keySet()) {
final Long endOffset = endOffsets.get(partition);
- if (endOffset != null && endOffset > 0) {
- restoreConsumer.seek(partition, endOffset - 1);
- } else {
+ if (endOffset == null || endOffset <= 0) {
restoreConsumer.seekToBeginning(Collections.singleton(partition));
seekToBeginningPartitions.add(partition);
}
}
windowedPartitionsRetention.keySet().removeAll(seekToBeginningPartitions);
- final ConsumerRecords<byte[], byte[]> polledRecords = restoreConsumer.poll(pollTime);
+ // probe back from the head for the newest data record: under EOS endOffset-1 is
+ // usually a control record, and the optimisation would degrade to seekToBeginning
+ final Map<TopicPartition, Long> latestTimestamps = new HashMap<>();
+ final Set<TopicPartition> unresolved = new HashSet<>(windowedPartitionsRetention.keySet());
+ runBackwardProbe(unresolved, latestTimestamps, beginningOffsets, endOffsets);
- seekByRetentionFromPolledRecords(polledRecords, windowedPartitionsRetention, seekToBeginningPartitions);
+ seekByRetention(latestTimestamps, windowedPartitionsRetention, seekToBeginningPartitions);
} catch (final TimeoutException e) {
log.debug("Could not seek by timestamp for changelog partitions {}, falling back to seek-to-beginning",
windowedPartitionsRetention.keySet(), e);
@@ -1120,6 +1157,9 @@ public class StoreChangelogReader implements ChangelogReader {
windowedPartitionsRetention.keySet(), e);
seekToBeginningPartitions.addAll(windowedPartitionsRetention.keySet());
} finally {
+ // in the finally, not after the probe: a lookup that times out leaves through a
+ // catch, and a probe that fails that way must arm the backoff like any other
+ recordProbeOutcomes(windowedPartitionsRetention, seekToBeginningPartitions);
restoreConsumer.pause(allAssigned);
final Set<TopicPartition> toResume = new HashSet<>(allAssigned);
toResume.removeAll(previouslyPaused);
@@ -1136,16 +1176,102 @@ public class StoreChangelogReader implements ChangelogReader {
}
}
- private void seekByRetentionFromPolledRecords(final ConsumerRecords<byte[], byte[]> polledRecords,
- final Map<TopicPartition, Long> windowedPartitionsRetention,
- final Set<TopicPartition> seekToBeginningPartitions) {
+ /**
+ * Reads back from each partition's end for the newest timestamp, to drive the retention-based
+ * seek. A window that cannot answer is widened; what the widest cannot answer for goes to log start.
+ */
+ private void runBackwardProbe(final Set<TopicPartition> unresolved,
+ final Map<TopicPartition, Long> latestTimestamps,
+ final Map<TopicPartition, Long> beginningOffsets,
+ final Map<TopicPartition, Long> endOffsets) {
+ long previousBack = 0L;
+ for (final long back : PROBE_WINDOWS) {
+ if (unresolved.isEmpty()) {
+ return;
+ }
+ seekProbePositions(unresolved, back, previousBack, beginningOffsets, endOffsets);
+ pollProbeWindow(unresolved, latestTimestamps);
+ previousBack = back;
+ }
+ }
+
+ /** Seeks each unresolved partition back by {@code back} offsets from its end. */
+ private void seekProbePositions(final Set<TopicPartition> unresolved,
+ final long back,
+ final long previousBack,
+ final Map<TopicPartition, Long> beginningOffsets,
+ final Map<TopicPartition, Long> endOffsets) {
+ // poll() updates fetch positions for the whole assignment and the restore consumer has
+ // auto.offset.reset=none, so every partition needs a position before the first poll
+ for (final TopicPartition partition : unresolved) {
+ final long begin = beginningOffsets.getOrDefault(partition, 0L);
+ final long end = endOffsets.get(partition);
+ final long target = Math.max(begin, end - back);
+ // a partition whose narrower window already reached log start has nothing further to
+ // show; re-seeking it would only discard the fetch that is about to answer
+ if (previousBack > 0L && target == Math.max(begin, end - previousBack)) {
+ continue;
+ }
+ restoreConsumer.seek(partition, target);
+ }
+ }
+
+ /** Polls the current window until it stops answering and has been given its dues. */
+ private void pollProbeWindow(final Set<TopicPartition> unresolved,
+ final Map<TopicPartition, Long> latestTimestamps) {
+ // a poll returns at most max.poll.records across all partitions, so one window can take
+ // several polls to reach them all; a fixed count abandons partitions still being served
+ final long startNs = time.nanoseconds();
+ final long minWaitNs = pollTime.toNanos() * PROBE_MIN_WAIT_POLLS;
+ int idlePolls = 0;
+ int polls = 0;
+ while (!unresolved.isEmpty() && polls < PROBE_MAX_POLLS
+ && (idlePolls < PROBE_IDLE_POLLS || time.nanoseconds() - startNs < minWaitNs)) {
+ final int remaining = unresolved.size();
+ collectProbed(restoreConsumer.poll(pollTime), unresolved, latestTimestamps);
+ polls++;
+ idlePolls = unresolved.size() < remaining ? 0 : idlePolls + 1;
+ }
+ }
+
+ /**
+ * Resolves any partition the poll answered for, taking the newest timestamp in the window: it
+ * estimates observed stream time, and a timestamp the log holds cannot overshoot it.
+ */
+ private void collectProbed(final ConsumerRecords<byte[], byte[]> probed,
+ final Set<TopicPartition> unresolved,
+ final Map<TopicPartition, Long> latestTimestamps) {
+ final Set<TopicPartition> resolved = new HashSet<>();
+ final Iterator<TopicPartition> iterator = unresolved.iterator();
+ while (iterator.hasNext()) {
+ final TopicPartition partition = iterator.next();
+ final List<ConsumerRecord<byte[], byte[]>> records = probed.records(partition);
+ if (records.isEmpty()) {
+ continue;
+ }
+ long latest = Long.MIN_VALUE;
+ for (final ConsumerRecord<byte[], byte[]> record : records) {
+ latest = Math.max(latest, record.timestamp());
+ }
+ latestTimestamps.put(partition, latest);
+ resolved.add(partition);
+ iterator.remove();
+ }
+ if (!resolved.isEmpty()) {
+ // stop resolved partitions competing for the next poll's max.poll.records budget
+ restoreConsumer.pause(resolved);
+ }
+ }
+
+ private void seekByRetention(final Map<TopicPartition, Long> latestTimestamps,
+ final Map<TopicPartition, Long> windowedPartitionsRetention,
+ final Set<TopicPartition> seekToBeginningPartitions) {
final Map<TopicPartition, Long> seekTimestamps = new HashMap<>();
for (final Map.Entry<TopicPartition, Long> entry : windowedPartitionsRetention.entrySet()) {
final TopicPartition partition = entry.getKey();
final long retentionPeriod = entry.getValue();
- final List<ConsumerRecord<byte[], byte[]>> records = polledRecords.records(partition);
- if (!records.isEmpty()) {
- final long latestTimestamp = records.get(0).timestamp();
+ final Long latestTimestamp = latestTimestamps.get(partition);
+ if (latestTimestamp != null) {
final long seekTimestamp = latestTimestamp - retentionPeriod;
if (seekTimestamp > 0) {
seekTimestamps.put(partition, seekTimestamp);
@@ -1171,6 +1297,24 @@ public class StoreChangelogReader implements ChangelogReader {
}
}
+ /** Arms the backoff for partitions the probe could not place, and clears it for those it did. */
+ private void recordProbeOutcomes(final Map<TopicPartition, Long> windowedPartitionsRetention,
+ final Set<TopicPartition> seekToBeginningPartitions) {
+ for (final TopicPartition partition : windowedPartitionsRetention.keySet()) {
+ if (seekToBeginningPartitions.contains(partition)) {
+ probeFailedAtMs.put(partition, time.milliseconds());
+ } else {
+ probeFailedAtMs.remove(partition);
+ }
+ }
+ }
+
+ /** A partition whose probe just fell back is left alone until the backoff expires. */
+ private boolean probeIsDue(final TopicPartition partition) {
+ final Long failedAt = probeFailedAtMs.get(partition);
+ return failedAt == null || time.milliseconds() - failedAt >= PROBE_RETRY_BACKOFF.toMillis();
+ }
+
@Override
public void unregister(final Collection<TopicPartition> revokedChangelogs) {
unregister(revokedChangelogs, StandbyUpdateListener.SuspendReason.MIGRATED);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
index d3a38ea8cb4..e68a2302d0d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
@@ -62,8 +62,10 @@ import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.time.Duration;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
@@ -1506,6 +1508,596 @@ public class StoreChangelogReaderTest {
}
}
+ /**
+ * The first poll after assignment returns nothing and every record arrives from the next poll
+ * on. An empty poll only means the fetch has not landed, so giving up on it sends partitions
+ * to a log-start seek with no margin against retention.
+ */
+ @Test
+ public void shouldRetryProbePollBeforeFallingBackToLogStart() {
+ final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+ final long beginOffset = 900_000L;
+ final long logEndOffset = 1_000_000L;
+ final long seekTarget = 999_000L;
+ final int numPartitions = 12; // more than one poll can plausibly serve at once
+
+ final TopicPartition[] tps = new TopicPartition[numPartitions];
+ final Map<TopicPartition, Long> begins = new HashMap<>();
+ final Map<TopicPartition, Long> ends = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ tps[i] = new TopicPartition(tp.topic(), i);
+ begins.put(tps[i], beginOffset);
+ ends.put(tps[i], logEndOffset);
+ }
+
+ // a position after restore reflects records since consumed, not where it was seeked, so
+ // seekToBeginning is the only unambiguous signal that the optimisation was abandoned
+ final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) ->
+ result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+ return result;
+ }
+
+ @Override
+ public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+ seekedToBeginning.addAll(partitions);
+ super.seekToBeginning(partitions);
+ }
+ };
+ probeConsumer.updateBeginningOffsets(begins);
+ probeConsumer.updateEndOffsets(ends);
+ adminClient.updateEndOffsets(ends);
+
+ // records can only be added once assigned, and earlier polls happen before that; the first
+ // poll after assignment delivers nothing, standing in for a fetch that has not landed
+ final int[] assignedPolls = {0};
+ for (int round = 0; round < numPartitions * 4; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (!probeConsumer.assignment().contains(tps[0])) {
+ return;
+ }
+ if (++assignedPolls[0] <= 1) {
+ return;
+ }
+ for (final TopicPartition partition : tps) {
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ partition.topic(), partition.partition(), logEndOffset - 1,
+ 10_000_000L, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ }
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ for (int i = 0; i < numPartitions; i++) {
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tps[i]);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L); // no checkpoint, then a value once restoring
+ when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tps[i])).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(new TaskId(0, i));
+ probeReader.register(tps[i], manager);
+ }
+
+ final Map<TaskId, Task> probeTasks = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ probeTasks.put(new TaskId(0, i), mock(Task.class));
+ }
+ probeReader.restore(probeTasks);
+
+ assertEquals(Collections.emptySet(), seekedToBeginning,
+ "every partition has a 3s retention against a 100k-record log, so the optimisation "
+ + "should apply to all of them; these were abandoned on one empty poll");
+ }
+
+ /**
+ * One partition is answered per poll, so resolving all of them takes more polls than any fixed
+ * budget in the old design allowed. A budget that stops while partitions are still being served
+ * sends them to a log-start restore with no margin against retention.
+ */
+ @Test
+ public void shouldKeepPollingWhileTheWindowIsStillResolvingPartitions() {
+ final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+ final long beginOffset = 900_000L;
+ final long logEndOffset = 1_000_000L;
+ final long seekTarget = 999_000L;
+ final int numPartitions = 20; // more polls than any fixed budget in the old design
+
+ final TopicPartition[] tps = new TopicPartition[numPartitions];
+ final Map<TopicPartition, Long> begins = new HashMap<>();
+ final Map<TopicPartition, Long> ends = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ tps[i] = new TopicPartition(tp.topic(), i);
+ begins.put(tps[i], beginOffset);
+ ends.put(tps[i], logEndOffset);
+ }
+
+ final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) ->
+ result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+ return result;
+ }
+
+ @Override
+ public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+ seekedToBeginning.addAll(partitions);
+ super.seekToBeginning(partitions);
+ }
+ };
+ probeConsumer.updateBeginningOffsets(begins);
+ probeConsumer.updateEndOffsets(ends);
+ adminClient.updateEndOffsets(ends);
+
+ // one partition per poll: the shape a shared max.poll.records budget produces when each
+ // window is large enough to fill a poll on its own
+ final int[] answered = {0};
+ for (int round = 0; round < numPartitions * 4; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (!probeConsumer.assignment().contains(tps[0]) || answered[0] >= numPartitions) {
+ return;
+ }
+ final TopicPartition partition = tps[answered[0]++];
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ partition.topic(), partition.partition(), logEndOffset - 1,
+ 10_000_000L, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ for (int i = 0; i < numPartitions; i++) {
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tps[i]);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L); // no checkpoint, then a value once restoring
+ when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tps[i])).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(new TaskId(0, i));
+ probeReader.register(tps[i], manager);
+ }
+
+ final Map<TaskId, Task> probeTasks = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ probeTasks.put(new TaskId(0, i), mock(Task.class));
+ }
+ probeReader.restore(probeTasks);
+
+ assertEquals(Collections.emptySet(), seekedToBeginning,
+ "every partition answered the probe, just not all in the same poll; these were "
+ + "abandoned by a budget that stopped while they were still being served");
+ }
+
+ /**
+ * A poll that returns without waiting has given no fetch a chance to land, so a run of them is
+ * no more evidence than none. Here ten polls deliver nothing while the clock does not move,
+ * which must not be read as an empty window.
+ */
+ @Test
+ public void shouldNotGiveUpOnPollsThatHaveNotWaited() {
+ final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+ final long beginOffset = 900_000L;
+ final long logEndOffset = 1_000_000L;
+ final long seekTarget = 999_000L;
+ final int numPartitions = 6;
+ final int silentPolls = 10; // well past PROBE_IDLE_POLLS, with time standing still
+
+ final TopicPartition[] tps = new TopicPartition[numPartitions];
+ final Map<TopicPartition, Long> begins = new HashMap<>();
+ final Map<TopicPartition, Long> ends = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ tps[i] = new TopicPartition(tp.topic(), i);
+ begins.put(tps[i], beginOffset);
+ ends.put(tps[i], logEndOffset);
+ }
+
+ final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) ->
+ result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+ return result;
+ }
+
+ @Override
+ public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+ seekedToBeginning.addAll(partitions);
+ super.seekToBeginning(partitions);
+ }
+ };
+ probeConsumer.updateBeginningOffsets(begins);
+ probeConsumer.updateEndOffsets(ends);
+ adminClient.updateEndOffsets(ends);
+
+ final int[] answered = {0};
+ for (int round = 0; round < 60; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (!probeConsumer.assignment().contains(tps[0]) || ++answered[0] <= silentPolls) {
+ return;
+ }
+ for (final TopicPartition partition : tps) {
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ partition.topic(), partition.partition(), logEndOffset - 1,
+ 10_000_000L, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ }
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ for (int i = 0; i < numPartitions; i++) {
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tps[i]);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L); // no checkpoint, then a value once restoring
+ when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tps[i])).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(new TaskId(0, i));
+ probeReader.register(tps[i], manager);
+ }
+
+ final Map<TaskId, Task> probeTasks = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ probeTasks.put(new TaskId(0, i), mock(Task.class));
+ }
+ probeReader.restore(probeTasks);
+
+ assertEquals(Collections.emptySet(), seekedToBeginning,
+ "no time passed while those polls came back empty, so none of them waited on a fetch; "
+ + "abandoning the window on that basis sends partitions to a log-start restore");
+ }
+
+ /**
+ * After a producer fence the restore consumer answers nothing for a while. Here twelve polls
+ * each block a full poll timeout and return empty before the window finally answers, which is
+ * shorter than the slowest probes measured on a soak, so none of it may be called an empty
+ * window.
+ */
+ @Test
+ public void shouldWaitOutAnUnreadyConsumerBeforeAbandoningTheWindow() {
+ final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+ final long beginOffset = 900_000L;
+ final long logEndOffset = 1_000_000L;
+ final long seekTarget = 999_000L;
+ final int numPartitions = 6;
+ final int unreadyPolls = 12; // 1.2s at the default poll.ms, all of it fruitless
+
+ final TopicPartition[] tps = new TopicPartition[numPartitions];
+ final Map<TopicPartition, Long> begins = new HashMap<>();
+ final Map<TopicPartition, Long> ends = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ tps[i] = new TopicPartition(tp.topic(), i);
+ begins.put(tps[i], beginOffset);
+ ends.put(tps[i], logEndOffset);
+ }
+
+ final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) ->
+ result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+ return result;
+ }
+
+ @Override
+ public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+ seekedToBeginning.addAll(partitions);
+ super.seekToBeginning(partitions);
+ }
+ };
+ probeConsumer.updateBeginningOffsets(begins);
+ probeConsumer.updateEndOffsets(ends);
+ adminClient.updateEndOffsets(ends);
+
+ final long pollMs = config.getLong(StreamsConfig.POLL_MS_CONFIG);
+ final int[] answered = {0};
+ for (int round = 0; round < 60; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (!probeConsumer.assignment().contains(tps[0])) {
+ return;
+ }
+ if (++answered[0] <= unreadyPolls) {
+ time.sleep(pollMs); // the poll blocked its full timeout and found nothing
+ return;
+ }
+ for (final TopicPartition partition : tps) {
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ partition.topic(), partition.partition(), logEndOffset - 1,
+ 10_000_000L, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ }
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ for (int i = 0; i < numPartitions; i++) {
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tps[i]);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L); // no checkpoint, then a value once restoring
+ when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tps[i])).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(new TaskId(0, i));
+ probeReader.register(tps[i], manager);
+ }
+
+ final Map<TaskId, Task> probeTasks = new HashMap<>();
+ for (int i = 0; i < numPartitions; i++) {
+ probeTasks.put(new TaskId(0, i), mock(Task.class));
+ }
+ probeReader.restore(probeTasks);
+
+ assertEquals(Collections.emptySet(), seekedToBeginning,
+ "the consumer was not ready to answer for 1.2s, which is inside the window the probe "
+ + "owes before it may conclude anything; these were abandoned early");
+ }
+
+ /**
+ * A task corrupted, wiped and re-registered with no offset would otherwise pay a full probe on
+ * every iteration. A probe that fell back is left alone until the backoff expires, which bounds
+ * a loop to one probe per interval without suppressing a probe that was working.
+ */
+ @Test
+ public void shouldBackOffProbingAPartitionWhoseProbeJustFailed() {
+ final long retentionMs = Duration.ofSeconds(3).toMillis();
+ final long endOffset = 1_000L;
+ final int[] probeRounds = {0};
+
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, Long> beginningOffsets(final Collection<TopicPartition> partitions) {
+ probeRounds[0]++; // only the probe looks these up
+ return super.beginningOffsets(partitions);
+ }
+ };
+ probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+ probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+ adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+ // no records are ever scheduled, so no window can answer and every probe falls back
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ final TaskId probeTaskId = new TaskId(0, 0);
+ final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+ for (int round = 0; round < 5; round++) {
+ registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+ }
+ assertEquals(1, probeRounds[0],
+ "the probe fell back, so re-registering in a loop should not probe again");
+
+ time.sleep(Duration.ofSeconds(60).toMillis()); // PROBE_RETRY_BACKOFF
+ registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+ assertEquals(2, probeRounds[0],
+ "once the backoff expires the partition is probed again, so a transient failure does "
+ + "not disable the optimisation for good");
+ }
+
+ private void registerRestoreAndRevoke(final StoreChangelogReader probeReader,
+ final TaskId probeTaskId,
+ final Map<TaskId, Task> probeTasks,
+ final long retentionMs) {
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tp);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L);
+ when(meta.retentionPeriod()).thenReturn(retentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tp)).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(probeTaskId);
+
+ probeReader.register(tp, manager);
+ probeReader.restore(probeTasks);
+ probeReader.unregister(Collections.singleton(tp));
+ }
+
+ /**
+ * A probe can also fail by the offset lookup timing out, which leaves through a catch rather
+ * than the normal path. That has to arm the backoff too, or a task looping on a slow broker
+ * re-probes on every iteration.
+ */
+ @Test
+ public void shouldBackOffWhenTheProbeFailsByTimeout() {
+ final long retentionMs = Duration.ofSeconds(3).toMillis();
+ final long endOffset = 1_000L;
+ final int[] lookups = {0};
+
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, Long> endOffsets(final Collection<TopicPartition> partitions) {
+ lookups[0]++;
+ throw new TimeoutException("timed out looking up end offsets");
+ }
+ };
+ probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+ probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+ adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ final TaskId probeTaskId = new TaskId(0, 0);
+ final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+ for (int round = 0; round < 4; round++) {
+ registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+ }
+
+ assertEquals(1, lookups[0],
+ "the probe failed by timeout, so re-registering in a loop should not probe again");
+ }
+
+ /**
+ * Only a probe that fell back arms the backoff. Suppressing a probe that was working would send
+ * the next restore to log start, which is what gets a partition lapped and corrupted in the
+ * first place -- the guard would sustain the loop it exists to bound.
+ */
+ @Test
+ public void shouldNotBackOffAfterAProbeThatSucceeded() {
+ final long retentionMs = Duration.ofSeconds(3).toMillis();
+ final long endOffset = 1_000L;
+ final int[] probeRounds = {0};
+
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, Long> beginningOffsets(final Collection<TopicPartition> partitions) {
+ probeRounds[0]++;
+ return super.beginningOffsets(partitions);
+ }
+
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) -> result.put(k, new OffsetAndTimestamp(1L, v)));
+ return result;
+ }
+ };
+ probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+ probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+ adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+
+ for (int round = 0; round < 40; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (probeConsumer.assignment().contains(tp)) {
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ tp.topic(), tp.partition(), endOffset - 1, 10_000_000L, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ }
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ final TaskId probeTaskId = new TaskId(0, 0);
+ final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+ // back to back, with no time passing at all
+ registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+ registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+
+ assertEquals(2, probeRounds[0],
+ "the probe answered, so nothing should be held back on the next registration");
+ }
+
+ /**
+ * The newest timestamp sits in the middle of the window, so taking the first or the last record
+ * in offset order picks the wrong one. The probe estimates observed stream time, a maximum.
+ */
+ @Test
+ public void shouldSeekFromTheNewestTimestampInTheProbedWindow() {
+ final long retentionMs = Duration.ofSeconds(30).toMillis();
+ final long logEndOffset = 1_000L;
+ final long oldest = 500_000L;
+ final long newest = 900_000L;
+ final long middle = 700_000L; // last in offset order, but not the newest
+
+ final Map<TopicPartition, Long> requested = new HashMap<>();
+ final MockConsumer<byte[], byte[]> probeConsumer =
+ new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+ @Override
+ public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+ final Map<TopicPartition, Long> timestampsToSearch) {
+ requested.putAll(timestampsToSearch);
+ final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+ timestampsToSearch.forEach((k, v) -> result.put(k, new OffsetAndTimestamp(1L, v)));
+ return result;
+ }
+ };
+ probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+ probeConsumer.updateEndOffsets(Collections.singletonMap(tp, logEndOffset));
+ adminClient.updateEndOffsets(Collections.singletonMap(tp, logEndOffset));
+
+ for (int round = 0; round < 8; round++) {
+ probeConsumer.schedulePollTask(() -> {
+ if (!probeConsumer.assignment().contains(tp)) {
+ return;
+ }
+ long offset = logEndOffset - 3;
+ for (final long timestamp : new long[] {oldest, newest, middle}) {
+ probeConsumer.addRecord(new ConsumerRecord<>(
+ tp.topic(), tp.partition(), offset++, timestamp, TimestampType.CREATE_TIME,
+ 0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+ }
+ });
+ }
+
+ final StoreChangelogReader probeReader = new StoreChangelogReader(
+ time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+ final TaskId probeTaskId = new TaskId(0, 0);
+ final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+ final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+ final StateStore store = mock(StateStore.class);
+ when(meta.changelogPartition()).thenReturn(tp);
+ when(meta.store()).thenReturn(store);
+ when(meta.offset()).thenReturn(null, 0L); // no checkpoint, then a value once restoring
+ when(meta.retentionPeriod()).thenReturn(retentionMs);
+ when(store.name()).thenReturn(storeName);
+ when(manager.storeMetadata(tp)).thenReturn(meta);
+ when(manager.taskType()).thenReturn(ACTIVE);
+ when(manager.taskId()).thenReturn(probeTaskId);
+ probeReader.register(tp, manager);
+
+ probeReader.restore(Collections.singletonMap(probeTaskId, mock(Task.class)));
+
+ assertEquals(newest - retentionMs, requested.get(tp).longValue(),
+ "the seek must be derived from the newest timestamp in the window (" + newest
+ + "), not the first or last record in offset order");
+ }
+
@Test
public void shouldSeekByTimestampForWindowedStoreWithoutCheckpoint() {
final long retentionMs = Duration.ofHours(2).toMillis();