You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by bb...@apache.org on 2026/09/08 17:27:05 UTC

(kafka) 03/04: KAFKA-20876: Windowed restore optimisation gives up when `endOffset-1` is a transaction control record (#23086)

This is an automated email from the ASF dual-hosted git repository.

bbejeck pushed a commit to branch 4.3
in repository https://gitbox.apache.org/repos/asf/kafka.git

commit b3add3983546158bd2eeaaf2fa62e8f1b68edb89
Author: Alan Lau <72...@users.noreply.github.com>
AuthorDate: Thu Aug 13 16:11:06 2026 -0400

    KAFKA-20876: Windowed restore optimisation gives up when `endOffset-1` is a transaction control record (#23086)
    
    Jira: https://issues.apache.org/jira/browse/KAFKA-20876
    
    When restoring a windowed store with no checkpoint,
    `StoreChangelogReader` seeks to `latestTimestamp - retentionPeriod`
    rather than log-start, since data older than the retention is discarded
    on write. It learns `latestTimestamp` by seeking to `endOffset - 1` and
    polling once in `seekNewPartitions`; an empty poll falls back to
    `seekToBeginning`.
    
    The probe now reads a window back from the end rather than a single
    offset, widens it when that window holds no record, and keeps polling a
    window for as long as it is still resolving partitions.
    
    Reviewers: Bill Bejeck <bb...@apache.org>
---
 .../processor/internals/StoreChangelogReader.java  | 168 +++++-
 .../internals/StoreChangelogReaderTest.java        | 592 +++++++++++++++++++++
 2 files changed, 748 insertions(+), 12 deletions(-)

diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
index e634aae279b..3c82dcc7259 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StoreChangelogReader.java
@@ -55,6 +55,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
@@ -205,8 +206,34 @@ public class StoreChangelogReader implements ChangelogReader {
 
     private static final long DEFAULT_OFFSET_UPDATE_MS = Duration.ofMinutes(5L).toMillis();
 
+    // Windows the probe reads back from each partition's end. Not 1: under EOS the last offset is
+    // usually a transaction control record, never delivered to a consumer, so probing there is a
+    // guaranteed empty poll.
+    private static final long[] PROBE_WINDOWS = {128L, 512L, 2048L};
+
+    // One empty poll proves nothing: poll() returns as soon as any fetch lands, so a partition can
+    // come back empty because another was served first.
+    private static final int PROBE_IDLE_POLLS = 3;
+
+    // Floor on the waiting a window is owed before it may be called empty: polls that return
+    // immediately have not waited on a fetch, and the floor has to cover a widening, whose re-seek
+    // discards the in-flight fetch and queues behind it -- one fetch in flight per broker.
+    private static final int PROBE_MIN_WAIT_POLLS = 10;
+
+    // Ceiling per window, so polls that return without waiting cannot spin the probe.
+    private static final int PROBE_MAX_POLLS = 30;
+
+    // How long a partition whose probe fell back is left alone, so a task corrupted, wiped and
+    // re-registered in a loop cannot probe on every iteration. Only a failed probe arms it, so a
+    // probe that is working is never suppressed.
+    private static final Duration PROBE_RETRY_BACKOFF = Duration.ofSeconds(60);
+
     private ChangelogReaderState state;
 
+    // deliberately outlives unregister: a changelog that is corrupted and re-registered in a loop
+    // must not forget that probing it has just failed
+    private final Map<TopicPartition, Long> probeFailedAtMs = new HashMap<>();
+
     private final Time time;
     private final Logger log;
     private final Duration pollTime;
@@ -1019,7 +1046,13 @@ public class StoreChangelogReader implements ChangelogReader {
             } else {
                 final long retentionPeriod = storeMetadata.retentionPeriod();
                 if (retentionPeriod > 0 && retentionPeriod != Long.MAX_VALUE) {
-                    newWindowedPartitionsRetention.put(partition, retentionPeriod);
+                    if (probeIsDue(partition)) {
+                        newWindowedPartitionsRetention.put(partition, retentionPeriod);
+                    } else {
+                        log.debug("Start restoring changelog partition {} from the beginning offset to end offset {} " +
+                            "since its last attempt to seek past expired data failed.", partition, recordEndOffset(endOffset));
+                        newSeekToBeginningPartitions.add(partition);
+                    }
                 } else {
                     final StateStore store = storeMetadata.store();
                     if (store instanceof WindowStore || store instanceof SessionStore) {
@@ -1096,21 +1129,25 @@ public class StoreChangelogReader implements ChangelogReader {
 
                 final Map<TopicPartition, Long> endOffsets =
                     restoreConsumer.endOffsets(windowedPartitionsRetention.keySet());
+                final Map<TopicPartition, Long> beginningOffsets =
+                    restoreConsumer.beginningOffsets(windowedPartitionsRetention.keySet());
 
                 for (final TopicPartition partition : windowedPartitionsRetention.keySet()) {
                     final Long endOffset = endOffsets.get(partition);
-                    if (endOffset != null && endOffset > 0) {
-                        restoreConsumer.seek(partition, endOffset - 1);
-                    } else {
+                    if (endOffset == null || endOffset <= 0) {
                         restoreConsumer.seekToBeginning(Collections.singleton(partition));
                         seekToBeginningPartitions.add(partition);
                     }
                 }
                 windowedPartitionsRetention.keySet().removeAll(seekToBeginningPartitions);
 
-                final ConsumerRecords<byte[], byte[]> polledRecords = restoreConsumer.poll(pollTime);
+                // probe back from the head for the newest data record: under EOS endOffset-1 is
+                // usually a control record, and the optimisation would degrade to seekToBeginning
+                final Map<TopicPartition, Long> latestTimestamps = new HashMap<>();
+                final Set<TopicPartition> unresolved = new HashSet<>(windowedPartitionsRetention.keySet());
+                runBackwardProbe(unresolved, latestTimestamps, beginningOffsets, endOffsets);
 
-                seekByRetentionFromPolledRecords(polledRecords, windowedPartitionsRetention, seekToBeginningPartitions);
+                seekByRetention(latestTimestamps, windowedPartitionsRetention, seekToBeginningPartitions);
             } catch (final TimeoutException e) {
                 log.debug("Could not seek by timestamp for changelog partitions {}, falling back to seek-to-beginning",
                     windowedPartitionsRetention.keySet(), e);
@@ -1120,6 +1157,9 @@ public class StoreChangelogReader implements ChangelogReader {
                     windowedPartitionsRetention.keySet(), e);
                 seekToBeginningPartitions.addAll(windowedPartitionsRetention.keySet());
             } finally {
+                // in the finally, not after the probe: a lookup that times out leaves through a
+                // catch, and a probe that fails that way must arm the backoff like any other
+                recordProbeOutcomes(windowedPartitionsRetention, seekToBeginningPartitions);
                 restoreConsumer.pause(allAssigned);
                 final Set<TopicPartition> toResume = new HashSet<>(allAssigned);
                 toResume.removeAll(previouslyPaused);
@@ -1136,16 +1176,102 @@ public class StoreChangelogReader implements ChangelogReader {
         }
     }
 
-    private void seekByRetentionFromPolledRecords(final ConsumerRecords<byte[], byte[]> polledRecords,
-                                                   final Map<TopicPartition, Long> windowedPartitionsRetention,
-                                                   final Set<TopicPartition> seekToBeginningPartitions) {
+    /**
+     * Reads back from each partition's end for the newest timestamp, to drive the retention-based
+     * seek. A window that cannot answer is widened; what the widest cannot answer for goes to log start.
+     */
+    private void runBackwardProbe(final Set<TopicPartition> unresolved,
+                                  final Map<TopicPartition, Long> latestTimestamps,
+                                  final Map<TopicPartition, Long> beginningOffsets,
+                                  final Map<TopicPartition, Long> endOffsets) {
+        long previousBack = 0L;
+        for (final long back : PROBE_WINDOWS) {
+            if (unresolved.isEmpty()) {
+                return;
+            }
+            seekProbePositions(unresolved, back, previousBack, beginningOffsets, endOffsets);
+            pollProbeWindow(unresolved, latestTimestamps);
+            previousBack = back;
+        }
+    }
+
+    /** Seeks each unresolved partition back by {@code back} offsets from its end. */
+    private void seekProbePositions(final Set<TopicPartition> unresolved,
+                                    final long back,
+                                    final long previousBack,
+                                    final Map<TopicPartition, Long> beginningOffsets,
+                                    final Map<TopicPartition, Long> endOffsets) {
+        // poll() updates fetch positions for the whole assignment and the restore consumer has
+        // auto.offset.reset=none, so every partition needs a position before the first poll
+        for (final TopicPartition partition : unresolved) {
+            final long begin = beginningOffsets.getOrDefault(partition, 0L);
+            final long end = endOffsets.get(partition);
+            final long target = Math.max(begin, end - back);
+            // a partition whose narrower window already reached log start has nothing further to
+            // show; re-seeking it would only discard the fetch that is about to answer
+            if (previousBack > 0L && target == Math.max(begin, end - previousBack)) {
+                continue;
+            }
+            restoreConsumer.seek(partition, target);
+        }
+    }
+
+    /** Polls the current window until it stops answering and has been given its dues. */
+    private void pollProbeWindow(final Set<TopicPartition> unresolved,
+                                 final Map<TopicPartition, Long> latestTimestamps) {
+        // a poll returns at most max.poll.records across all partitions, so one window can take
+        // several polls to reach them all; a fixed count abandons partitions still being served
+        final long startNs = time.nanoseconds();
+        final long minWaitNs = pollTime.toNanos() * PROBE_MIN_WAIT_POLLS;
+        int idlePolls = 0;
+        int polls = 0;
+        while (!unresolved.isEmpty() && polls < PROBE_MAX_POLLS
+            && (idlePolls < PROBE_IDLE_POLLS || time.nanoseconds() - startNs < minWaitNs)) {
+            final int remaining = unresolved.size();
+            collectProbed(restoreConsumer.poll(pollTime), unresolved, latestTimestamps);
+            polls++;
+            idlePolls = unresolved.size() < remaining ? 0 : idlePolls + 1;
+        }
+    }
+
+    /**
+     * Resolves any partition the poll answered for, taking the newest timestamp in the window: it
+     * estimates observed stream time, and a timestamp the log holds cannot overshoot it.
+     */
+    private void collectProbed(final ConsumerRecords<byte[], byte[]> probed,
+                               final Set<TopicPartition> unresolved,
+                               final Map<TopicPartition, Long> latestTimestamps) {
+        final Set<TopicPartition> resolved = new HashSet<>();
+        final Iterator<TopicPartition> iterator = unresolved.iterator();
+        while (iterator.hasNext()) {
+            final TopicPartition partition = iterator.next();
+            final List<ConsumerRecord<byte[], byte[]>> records = probed.records(partition);
+            if (records.isEmpty()) {
+                continue;
+            }
+            long latest = Long.MIN_VALUE;
+            for (final ConsumerRecord<byte[], byte[]> record : records) {
+                latest = Math.max(latest, record.timestamp());
+            }
+            latestTimestamps.put(partition, latest);
+            resolved.add(partition);
+            iterator.remove();
+        }
+        if (!resolved.isEmpty()) {
+            // stop resolved partitions competing for the next poll's max.poll.records budget
+            restoreConsumer.pause(resolved);
+        }
+    }
+
+    private void seekByRetention(final Map<TopicPartition, Long> latestTimestamps,
+                                 final Map<TopicPartition, Long> windowedPartitionsRetention,
+                                 final Set<TopicPartition> seekToBeginningPartitions) {
         final Map<TopicPartition, Long> seekTimestamps = new HashMap<>();
         for (final Map.Entry<TopicPartition, Long> entry : windowedPartitionsRetention.entrySet()) {
             final TopicPartition partition = entry.getKey();
             final long retentionPeriod = entry.getValue();
-            final List<ConsumerRecord<byte[], byte[]>> records = polledRecords.records(partition);
-            if (!records.isEmpty()) {
-                final long latestTimestamp = records.get(0).timestamp();
+            final Long latestTimestamp = latestTimestamps.get(partition);
+            if (latestTimestamp != null) {
                 final long seekTimestamp = latestTimestamp - retentionPeriod;
                 if (seekTimestamp > 0) {
                     seekTimestamps.put(partition, seekTimestamp);
@@ -1171,6 +1297,24 @@ public class StoreChangelogReader implements ChangelogReader {
         }
     }
 
+    /** Arms the backoff for partitions the probe could not place, and clears it for those it did. */
+    private void recordProbeOutcomes(final Map<TopicPartition, Long> windowedPartitionsRetention,
+                                     final Set<TopicPartition> seekToBeginningPartitions) {
+        for (final TopicPartition partition : windowedPartitionsRetention.keySet()) {
+            if (seekToBeginningPartitions.contains(partition)) {
+                probeFailedAtMs.put(partition, time.milliseconds());
+            } else {
+                probeFailedAtMs.remove(partition);
+            }
+        }
+    }
+
+    /** A partition whose probe just fell back is left alone until the backoff expires. */
+    private boolean probeIsDue(final TopicPartition partition) {
+        final Long failedAt = probeFailedAtMs.get(partition);
+        return failedAt == null || time.milliseconds() - failedAt >= PROBE_RETRY_BACKOFF.toMillis();
+    }
+
     @Override
     public void unregister(final Collection<TopicPartition> revokedChangelogs) {
         unregister(revokedChangelogs, StandbyUpdateListener.SuspendReason.MIGRATED);
diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
index d3a38ea8cb4..e68a2302d0d 100644
--- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StoreChangelogReaderTest.java
@@ -62,8 +62,10 @@ import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
 
 import java.time.Duration;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
@@ -1506,6 +1508,596 @@ public class StoreChangelogReaderTest {
         }
     }
 
+    /**
+     * The first poll after assignment returns nothing and every record arrives from the next poll
+     * on. An empty poll only means the fetch has not landed, so giving up on it sends partitions
+     * to a log-start seek with no margin against retention.
+     */
+    @Test
+    public void shouldRetryProbePollBeforeFallingBackToLogStart() {
+        final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+        final long beginOffset = 900_000L;
+        final long logEndOffset = 1_000_000L;
+        final long seekTarget = 999_000L;
+        final int numPartitions = 12;      // more than one poll can plausibly serve at once
+
+        final TopicPartition[] tps = new TopicPartition[numPartitions];
+        final Map<TopicPartition, Long> begins = new HashMap<>();
+        final Map<TopicPartition, Long> ends = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            tps[i] = new TopicPartition(tp.topic(), i);
+            begins.put(tps[i], beginOffset);
+            ends.put(tps[i], logEndOffset);
+        }
+
+        // a position after restore reflects records since consumed, not where it was seeked, so
+        // seekToBeginning is the only unambiguous signal that the optimisation was abandoned
+        final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) ->
+                        result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+                    return result;
+                }
+
+                @Override
+                public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+                    seekedToBeginning.addAll(partitions);
+                    super.seekToBeginning(partitions);
+                }
+            };
+        probeConsumer.updateBeginningOffsets(begins);
+        probeConsumer.updateEndOffsets(ends);
+        adminClient.updateEndOffsets(ends);
+
+        // records can only be added once assigned, and earlier polls happen before that; the first
+        // poll after assignment delivers nothing, standing in for a fetch that has not landed
+        final int[] assignedPolls = {0};
+        for (int round = 0; round < numPartitions * 4; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (!probeConsumer.assignment().contains(tps[0])) {
+                    return;
+                }
+                if (++assignedPolls[0] <= 1) {
+                    return;
+                }
+                for (final TopicPartition partition : tps) {
+                    probeConsumer.addRecord(new ConsumerRecord<>(
+                        partition.topic(), partition.partition(), logEndOffset - 1,
+                        10_000_000L, TimestampType.CREATE_TIME,
+                        0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+                }
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        for (int i = 0; i < numPartitions; i++) {
+            final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+            final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+            final StateStore store = mock(StateStore.class);
+            when(meta.changelogPartition()).thenReturn(tps[i]);
+            when(meta.store()).thenReturn(store);
+            when(meta.offset()).thenReturn(null, 0L);   // no checkpoint, then a value once restoring
+            when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+            when(store.name()).thenReturn(storeName);
+            when(manager.storeMetadata(tps[i])).thenReturn(meta);
+            when(manager.taskType()).thenReturn(ACTIVE);
+            when(manager.taskId()).thenReturn(new TaskId(0, i));
+            probeReader.register(tps[i], manager);
+        }
+
+        final Map<TaskId, Task> probeTasks = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            probeTasks.put(new TaskId(0, i), mock(Task.class));
+        }
+        probeReader.restore(probeTasks);
+
+        assertEquals(Collections.emptySet(), seekedToBeginning,
+            "every partition has a 3s retention against a 100k-record log, so the optimisation "
+                + "should apply to all of them; these were abandoned on one empty poll");
+    }
+
+    /**
+     * One partition is answered per poll, so resolving all of them takes more polls than any fixed
+     * budget in the old design allowed. A budget that stops while partitions are still being served
+     * sends them to a log-start restore with no margin against retention.
+     */
+    @Test
+    public void shouldKeepPollingWhileTheWindowIsStillResolvingPartitions() {
+        final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+        final long beginOffset = 900_000L;
+        final long logEndOffset = 1_000_000L;
+        final long seekTarget = 999_000L;
+        final int numPartitions = 20;      // more polls than any fixed budget in the old design
+
+        final TopicPartition[] tps = new TopicPartition[numPartitions];
+        final Map<TopicPartition, Long> begins = new HashMap<>();
+        final Map<TopicPartition, Long> ends = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            tps[i] = new TopicPartition(tp.topic(), i);
+            begins.put(tps[i], beginOffset);
+            ends.put(tps[i], logEndOffset);
+        }
+
+        final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) ->
+                        result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+                    return result;
+                }
+
+                @Override
+                public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+                    seekedToBeginning.addAll(partitions);
+                    super.seekToBeginning(partitions);
+                }
+            };
+        probeConsumer.updateBeginningOffsets(begins);
+        probeConsumer.updateEndOffsets(ends);
+        adminClient.updateEndOffsets(ends);
+
+        // one partition per poll: the shape a shared max.poll.records budget produces when each
+        // window is large enough to fill a poll on its own
+        final int[] answered = {0};
+        for (int round = 0; round < numPartitions * 4; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (!probeConsumer.assignment().contains(tps[0]) || answered[0] >= numPartitions) {
+                    return;
+                }
+                final TopicPartition partition = tps[answered[0]++];
+                probeConsumer.addRecord(new ConsumerRecord<>(
+                    partition.topic(), partition.partition(), logEndOffset - 1,
+                    10_000_000L, TimestampType.CREATE_TIME,
+                    0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        for (int i = 0; i < numPartitions; i++) {
+            final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+            final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+            final StateStore store = mock(StateStore.class);
+            when(meta.changelogPartition()).thenReturn(tps[i]);
+            when(meta.store()).thenReturn(store);
+            when(meta.offset()).thenReturn(null, 0L);   // no checkpoint, then a value once restoring
+            when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+            when(store.name()).thenReturn(storeName);
+            when(manager.storeMetadata(tps[i])).thenReturn(meta);
+            when(manager.taskType()).thenReturn(ACTIVE);
+            when(manager.taskId()).thenReturn(new TaskId(0, i));
+            probeReader.register(tps[i], manager);
+        }
+
+        final Map<TaskId, Task> probeTasks = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            probeTasks.put(new TaskId(0, i), mock(Task.class));
+        }
+        probeReader.restore(probeTasks);
+
+        assertEquals(Collections.emptySet(), seekedToBeginning,
+            "every partition answered the probe, just not all in the same poll; these were "
+                + "abandoned by a budget that stopped while they were still being served");
+    }
+
+    /**
+     * A poll that returns without waiting has given no fetch a chance to land, so a run of them is
+     * no more evidence than none. Here ten polls deliver nothing while the clock does not move,
+     * which must not be read as an empty window.
+     */
+    @Test
+    public void shouldNotGiveUpOnPollsThatHaveNotWaited() {
+        final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+        final long beginOffset = 900_000L;
+        final long logEndOffset = 1_000_000L;
+        final long seekTarget = 999_000L;
+        final int numPartitions = 6;
+        final int silentPolls = 10;        // well past PROBE_IDLE_POLLS, with time standing still
+
+        final TopicPartition[] tps = new TopicPartition[numPartitions];
+        final Map<TopicPartition, Long> begins = new HashMap<>();
+        final Map<TopicPartition, Long> ends = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            tps[i] = new TopicPartition(tp.topic(), i);
+            begins.put(tps[i], beginOffset);
+            ends.put(tps[i], logEndOffset);
+        }
+
+        final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) ->
+                        result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+                    return result;
+                }
+
+                @Override
+                public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+                    seekedToBeginning.addAll(partitions);
+                    super.seekToBeginning(partitions);
+                }
+            };
+        probeConsumer.updateBeginningOffsets(begins);
+        probeConsumer.updateEndOffsets(ends);
+        adminClient.updateEndOffsets(ends);
+
+        final int[] answered = {0};
+        for (int round = 0; round < 60; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (!probeConsumer.assignment().contains(tps[0]) || ++answered[0] <= silentPolls) {
+                    return;
+                }
+                for (final TopicPartition partition : tps) {
+                    probeConsumer.addRecord(new ConsumerRecord<>(
+                        partition.topic(), partition.partition(), logEndOffset - 1,
+                        10_000_000L, TimestampType.CREATE_TIME,
+                        0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+                }
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        for (int i = 0; i < numPartitions; i++) {
+            final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+            final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+            final StateStore store = mock(StateStore.class);
+            when(meta.changelogPartition()).thenReturn(tps[i]);
+            when(meta.store()).thenReturn(store);
+            when(meta.offset()).thenReturn(null, 0L);   // no checkpoint, then a value once restoring
+            when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+            when(store.name()).thenReturn(storeName);
+            when(manager.storeMetadata(tps[i])).thenReturn(meta);
+            when(manager.taskType()).thenReturn(ACTIVE);
+            when(manager.taskId()).thenReturn(new TaskId(0, i));
+            probeReader.register(tps[i], manager);
+        }
+
+        final Map<TaskId, Task> probeTasks = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            probeTasks.put(new TaskId(0, i), mock(Task.class));
+        }
+        probeReader.restore(probeTasks);
+
+        assertEquals(Collections.emptySet(), seekedToBeginning,
+            "no time passed while those polls came back empty, so none of them waited on a fetch; "
+                + "abandoning the window on that basis sends partitions to a log-start restore");
+    }
+
+    /**
+     * After a producer fence the restore consumer answers nothing for a while. Here twelve polls
+     * each block a full poll timeout and return empty before the window finally answers, which is
+     * shorter than the slowest probes measured on a soak, so none of it may be called an empty
+     * window.
+     */
+    @Test
+    public void shouldWaitOutAnUnreadyConsumerBeforeAbandoningTheWindow() {
+        final long shortRetentionMs = Duration.ofSeconds(3).toMillis();
+        final long beginOffset = 900_000L;
+        final long logEndOffset = 1_000_000L;
+        final long seekTarget = 999_000L;
+        final int numPartitions = 6;
+        final int unreadyPolls = 12;       // 1.2s at the default poll.ms, all of it fruitless
+
+        final TopicPartition[] tps = new TopicPartition[numPartitions];
+        final Map<TopicPartition, Long> begins = new HashMap<>();
+        final Map<TopicPartition, Long> ends = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            tps[i] = new TopicPartition(tp.topic(), i);
+            begins.put(tps[i], beginOffset);
+            ends.put(tps[i], logEndOffset);
+        }
+
+        final Set<TopicPartition> seekedToBeginning = new HashSet<>();
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) ->
+                        result.put(k, new OffsetAndTimestamp(seekTarget, v)));
+                    return result;
+                }
+
+                @Override
+                public synchronized void seekToBeginning(final Collection<TopicPartition> partitions) {
+                    seekedToBeginning.addAll(partitions);
+                    super.seekToBeginning(partitions);
+                }
+            };
+        probeConsumer.updateBeginningOffsets(begins);
+        probeConsumer.updateEndOffsets(ends);
+        adminClient.updateEndOffsets(ends);
+
+        final long pollMs = config.getLong(StreamsConfig.POLL_MS_CONFIG);
+        final int[] answered = {0};
+        for (int round = 0; round < 60; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (!probeConsumer.assignment().contains(tps[0])) {
+                    return;
+                }
+                if (++answered[0] <= unreadyPolls) {
+                    time.sleep(pollMs);      // the poll blocked its full timeout and found nothing
+                    return;
+                }
+                for (final TopicPartition partition : tps) {
+                    probeConsumer.addRecord(new ConsumerRecord<>(
+                        partition.topic(), partition.partition(), logEndOffset - 1,
+                        10_000_000L, TimestampType.CREATE_TIME,
+                        0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+                }
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        for (int i = 0; i < numPartitions; i++) {
+            final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+            final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+            final StateStore store = mock(StateStore.class);
+            when(meta.changelogPartition()).thenReturn(tps[i]);
+            when(meta.store()).thenReturn(store);
+            when(meta.offset()).thenReturn(null, 0L);   // no checkpoint, then a value once restoring
+            when(meta.retentionPeriod()).thenReturn(shortRetentionMs);
+            when(store.name()).thenReturn(storeName);
+            when(manager.storeMetadata(tps[i])).thenReturn(meta);
+            when(manager.taskType()).thenReturn(ACTIVE);
+            when(manager.taskId()).thenReturn(new TaskId(0, i));
+            probeReader.register(tps[i], manager);
+        }
+
+        final Map<TaskId, Task> probeTasks = new HashMap<>();
+        for (int i = 0; i < numPartitions; i++) {
+            probeTasks.put(new TaskId(0, i), mock(Task.class));
+        }
+        probeReader.restore(probeTasks);
+
+        assertEquals(Collections.emptySet(), seekedToBeginning,
+            "the consumer was not ready to answer for 1.2s, which is inside the window the probe "
+                + "owes before it may conclude anything; these were abandoned early");
+    }
+
+    /**
+     * A task corrupted, wiped and re-registered with no offset would otherwise pay a full probe on
+     * every iteration. A probe that fell back is left alone until the backoff expires, which bounds
+     * a loop to one probe per interval without suppressing a probe that was working.
+     */
+    @Test
+    public void shouldBackOffProbingAPartitionWhoseProbeJustFailed() {
+        final long retentionMs = Duration.ofSeconds(3).toMillis();
+        final long endOffset = 1_000L;
+        final int[] probeRounds = {0};
+
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, Long> beginningOffsets(final Collection<TopicPartition> partitions) {
+                    probeRounds[0]++;      // only the probe looks these up
+                    return super.beginningOffsets(partitions);
+                }
+            };
+        probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+        probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+        adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+        // no records are ever scheduled, so no window can answer and every probe falls back
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        final TaskId probeTaskId = new TaskId(0, 0);
+        final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+        for (int round = 0; round < 5; round++) {
+            registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+        }
+        assertEquals(1, probeRounds[0],
+            "the probe fell back, so re-registering in a loop should not probe again");
+
+        time.sleep(Duration.ofSeconds(60).toMillis());     // PROBE_RETRY_BACKOFF
+        registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+        assertEquals(2, probeRounds[0],
+            "once the backoff expires the partition is probed again, so a transient failure does "
+                + "not disable the optimisation for good");
+    }
+
+    private void registerRestoreAndRevoke(final StoreChangelogReader probeReader,
+                                          final TaskId probeTaskId,
+                                          final Map<TaskId, Task> probeTasks,
+                                          final long retentionMs) {
+        final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+        final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+        final StateStore store = mock(StateStore.class);
+        when(meta.changelogPartition()).thenReturn(tp);
+        when(meta.store()).thenReturn(store);
+        when(meta.offset()).thenReturn(null, 0L);
+        when(meta.retentionPeriod()).thenReturn(retentionMs);
+        when(store.name()).thenReturn(storeName);
+        when(manager.storeMetadata(tp)).thenReturn(meta);
+        when(manager.taskType()).thenReturn(ACTIVE);
+        when(manager.taskId()).thenReturn(probeTaskId);
+
+        probeReader.register(tp, manager);
+        probeReader.restore(probeTasks);
+        probeReader.unregister(Collections.singleton(tp));
+    }
+
+    /**
+     * A probe can also fail by the offset lookup timing out, which leaves through a catch rather
+     * than the normal path. That has to arm the backoff too, or a task looping on a slow broker
+     * re-probes on every iteration.
+     */
+    @Test
+    public void shouldBackOffWhenTheProbeFailsByTimeout() {
+        final long retentionMs = Duration.ofSeconds(3).toMillis();
+        final long endOffset = 1_000L;
+        final int[] lookups = {0};
+
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, Long> endOffsets(final Collection<TopicPartition> partitions) {
+                    lookups[0]++;
+                    throw new TimeoutException("timed out looking up end offsets");
+                }
+            };
+        probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+        probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+        adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        final TaskId probeTaskId = new TaskId(0, 0);
+        final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+        for (int round = 0; round < 4; round++) {
+            registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+        }
+
+        assertEquals(1, lookups[0],
+            "the probe failed by timeout, so re-registering in a loop should not probe again");
+    }
+
+    /**
+     * Only a probe that fell back arms the backoff. Suppressing a probe that was working would send
+     * the next restore to log start, which is what gets a partition lapped and corrupted in the
+     * first place -- the guard would sustain the loop it exists to bound.
+     */
+    @Test
+    public void shouldNotBackOffAfterAProbeThatSucceeded() {
+        final long retentionMs = Duration.ofSeconds(3).toMillis();
+        final long endOffset = 1_000L;
+        final int[] probeRounds = {0};
+
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, Long> beginningOffsets(final Collection<TopicPartition> partitions) {
+                    probeRounds[0]++;
+                    return super.beginningOffsets(partitions);
+                }
+
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) -> result.put(k, new OffsetAndTimestamp(1L, v)));
+                    return result;
+                }
+            };
+        probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+        probeConsumer.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+        adminClient.updateEndOffsets(Collections.singletonMap(tp, endOffset));
+
+        for (int round = 0; round < 40; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (probeConsumer.assignment().contains(tp)) {
+                    probeConsumer.addRecord(new ConsumerRecord<>(
+                        tp.topic(), tp.partition(), endOffset - 1, 10_000_000L, TimestampType.CREATE_TIME,
+                        0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+                }
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        final TaskId probeTaskId = new TaskId(0, 0);
+        final Map<TaskId, Task> probeTasks = Collections.singletonMap(probeTaskId, mock(Task.class));
+
+        // back to back, with no time passing at all
+        registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+        registerRestoreAndRevoke(probeReader, probeTaskId, probeTasks, retentionMs);
+
+        assertEquals(2, probeRounds[0],
+            "the probe answered, so nothing should be held back on the next registration");
+    }
+
+    /**
+     * The newest timestamp sits in the middle of the window, so taking the first or the last record
+     * in offset order picks the wrong one. The probe estimates observed stream time, a maximum.
+     */
+    @Test
+    public void shouldSeekFromTheNewestTimestampInTheProbedWindow() {
+        final long retentionMs = Duration.ofSeconds(30).toMillis();
+        final long logEndOffset = 1_000L;
+        final long oldest = 500_000L;
+        final long newest = 900_000L;
+        final long middle = 700_000L;    // last in offset order, but not the newest
+
+        final Map<TopicPartition, Long> requested = new HashMap<>();
+        final MockConsumer<byte[], byte[]> probeConsumer =
+            new MockConsumer<>(AutoOffsetResetStrategy.EARLIEST.name()) {
+                @Override
+                public synchronized Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(
+                        final Map<TopicPartition, Long> timestampsToSearch) {
+                    requested.putAll(timestampsToSearch);
+                    final Map<TopicPartition, OffsetAndTimestamp> result = new HashMap<>();
+                    timestampsToSearch.forEach((k, v) -> result.put(k, new OffsetAndTimestamp(1L, v)));
+                    return result;
+                }
+            };
+        probeConsumer.updateBeginningOffsets(Collections.singletonMap(tp, 0L));
+        probeConsumer.updateEndOffsets(Collections.singletonMap(tp, logEndOffset));
+        adminClient.updateEndOffsets(Collections.singletonMap(tp, logEndOffset));
+
+        for (int round = 0; round < 8; round++) {
+            probeConsumer.schedulePollTask(() -> {
+                if (!probeConsumer.assignment().contains(tp)) {
+                    return;
+                }
+                long offset = logEndOffset - 3;
+                for (final long timestamp : new long[] {oldest, newest, middle}) {
+                    probeConsumer.addRecord(new ConsumerRecord<>(
+                        tp.topic(), tp.partition(), offset++, timestamp, TimestampType.CREATE_TIME,
+                        0, 0, new byte[0], new byte[0], new RecordHeaders(), Optional.empty()));
+                }
+            });
+        }
+
+        final StoreChangelogReader probeReader = new StoreChangelogReader(
+            time, config, logContext, adminClient, probeConsumer, callback, standbyListener);
+
+        final TaskId probeTaskId = new TaskId(0, 0);
+        final StateStoreMetadata meta = mock(StateStoreMetadata.class);
+        final ProcessorStateManager manager = mock(ProcessorStateManager.class);
+        final StateStore store = mock(StateStore.class);
+        when(meta.changelogPartition()).thenReturn(tp);
+        when(meta.store()).thenReturn(store);
+        when(meta.offset()).thenReturn(null, 0L);   // no checkpoint, then a value once restoring
+        when(meta.retentionPeriod()).thenReturn(retentionMs);
+        when(store.name()).thenReturn(storeName);
+        when(manager.storeMetadata(tp)).thenReturn(meta);
+        when(manager.taskType()).thenReturn(ACTIVE);
+        when(manager.taskId()).thenReturn(probeTaskId);
+        probeReader.register(tp, manager);
+
+        probeReader.restore(Collections.singletonMap(probeTaskId, mock(Task.class)));
+
+        assertEquals(newest - retentionMs, requested.get(tp).longValue(),
+            "the seek must be derived from the newest timestamp in the window (" + newest
+                + "), not the first or last record in offset order");
+    }
+
     @Test
     public void shouldSeekByTimestampForWindowedStoreWithoutCheckpoint() {
         final long retentionMs = Duration.ofHours(2).toMillis();