You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bd...@apache.org on 2026/04/14 21:46:59 UTC

(cassandra) 10/36: Implement unified log reconciliation

This is an automated email from the ASF dual-hosted git repository.

bdeggleston pushed a commit to branch cep-45-mutation-tracking
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 9676c0eaeb265094f0f679733ab3de819ca61646
Author: Aleksey Yeschenko <al...@apache.org>
AuthorDate: Fri May 16 11:35:32 2025 +0100

    Implement unified log reconciliation
    
    patch by Aleksey Yeschenko; reviewed by Blake Eggleston for
    CASSANDRA-20729
    
    Co-authored-by: Aleksey Yeschenko <al...@apache.org>
    Co-authored-by: Blake Eggleston <bl...@ultrablake.com>
---
 .../db/AbstractReadCommandVerbHandler.java         |   9 +
 src/java/org/apache/cassandra/db/Keyspace.java     |  24 +-
 .../apache/cassandra/db/MutationVerbHandler.java   |   1 +
 src/java/org/apache/cassandra/dht/Splitter.java    |   2 +-
 .../apache/cassandra/journal/ActiveSegment.java    |  15 +-
 .../apache/cassandra/journal/EntrySerializer.java  |  10 +
 src/java/org/apache/cassandra/journal/Flusher.java |   8 +-
 src/java/org/apache/cassandra/journal/Journal.java |  42 ++
 .../apache/cassandra/journal/RecordPointer.java    |  14 +-
 src/java/org/apache/cassandra/journal/Segment.java |  11 +
 .../org/apache/cassandra/journal/Segments.java     |  13 +-
 .../cassandra/locator/EndpointsForRange.java       |   5 +
 src/java/org/apache/cassandra/net/Message.java     |  11 +-
 src/java/org/apache/cassandra/net/Verb.java        |  37 +-
 .../cassandra/replication/ActiveLogReconciler.java | 207 +++++++
 ...icatedOffsets.java => BroadcastLogOffsets.java} |  20 +-
 .../cassandra/replication/CoordinatorLog.java      | 130 +++--
 .../cassandra/replication/ExpiredStatePurger.java  | 112 ++++
 .../cassandra/replication/ForwardedWrite.java      | 236 +++-----
 .../cassandra/replication/IncomingMutations.java   | 158 ++++++
 .../cassandra/replication/Log2OffsetsMap.java      |  75 ++-
 .../cassandra/replication/MutationJournal.java     |  21 +-
 .../cassandra/replication/MutationSummary.java     |  17 +
 .../replication/MutationTrackingService.java       | 228 ++++++--
 ...conciledMutations.java => Node2OffsetsMap.java} |  35 +-
 .../org/apache/cassandra/replication/Offsets.java  |  74 ++-
 .../cassandra/replication/OutgoingMutations.java   | 154 +++++
 .../replication/PullMutationsRequest.java          |  75 +++
 .../cassandra/replication/PushMutationRequest.java | 183 ++++++
 .../cassandra/replication/ReconciliationPlan.java  | 210 -------
 .../org/apache/cassandra/replication/Shard.java    |  89 ++-
 .../cassandra/replication/TrackedWriteRequest.java |  36 +-
 .../replication/UnreconciledMutations.java         | 160 +++++-
 .../replication/UnreconciledMutationsReplica.java  | 193 -------
 .../org/apache/cassandra/service/StorageProxy.java |   5 +-
 .../service/TrackedWriteResponseHandler.java       |  16 +-
 .../service/reads/range/RangeCommandIterator.java  |   6 +-
 .../reads/tracked/ExtendingCompletedRead.java      |  12 +-
 .../reads/tracked/FilteredFollowupRead.java        |  17 +-
 .../reads/tracked/PartialTrackedRangeRead.java     |  13 +-
 .../service/reads/tracked/PartialTrackedRead.java  |  23 +-
 .../tracked/PartialTrackedSinglePartitionRead.java |   2 +-
 ...dReconcileNotify.java => ReadReconcileAck.java} |  48 +-
 .../reads/tracked/ReadReconcileReceive.java        | 137 -----
 .../service/reads/tracked/ReadReconcileSend.java   | 166 ------
 .../service/reads/tracked/ReadReconciliations.java | 396 +++++++++++++
 .../reads/tracked/TrackedLocalReadCoordinator.java | 617 ---------------------
 .../service/reads/tracked/TrackedLocalReads.java   | 272 +++++----
 .../service/reads/tracked/TrackedRead.java         | 160 +++---
 .../reads/tracked/TrackedSummaryResponse.java      |  49 +-
 .../distributed/test/ReadRepairTestBase.java       |  20 +-
 .../distributed/test/ReadRepairTester.java         |   9 +-
 .../tracking/MutationTrackingPendingReadTest.java  |   4 +-
 .../test/tracking/MutationTrackingTest.java        |  58 +-
 .../replication/CoordinatorLogOffsetsTest.java     |  12 +-
 .../apache/cassandra/replication/OffsetsTest.java  |  95 +++-
 56 files changed, 2758 insertions(+), 1994 deletions(-)

diff --git a/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
index 9647891f8a..24e9a88974 100644
--- a/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
@@ -34,9 +34,11 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
 public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<T>
 {
@@ -46,6 +48,13 @@ public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<
 
     public final void doVerb(Message<T> message)
     {
+        if (approxTime.now() > message.expiresAtNanos())
+        {
+            Tracing.trace("Discarding read from {} (timed out)", message.from());
+            MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+            return;
+        }
+
         ClusterMetadata metadata = ClusterMetadata.current();
         if (message.epoch().isAfter(Epoch.EMPTY))
         {
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 532b534aa5..d6d0e45eac 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -615,24 +615,30 @@ public class Keyspace
         if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS))
             throw new RuntimeException("Testing write failures");
 
+        boolean started;
         try (WriteContext ctx = trackedWriteHandler.beginWrite(mutation, true))
         {
-            MutationTrackingService.instance.startWriting(mutation);
+            started = MutationTrackingService.instance.startWriting(mutation);
 
-            for (PartitionUpdate upd : mutation.getPartitionUpdates())
+            if (started)
             {
-                ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(upd.metadata().id);
-                if (cfs == null)
+                for (PartitionUpdate upd : mutation.getPartitionUpdates())
                 {
-                    logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name);
-                    continue;
-                }
+                    ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(upd.metadata().id);
+                    if (cfs == null)
+                    {
+                        logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name);
+                        continue;
+                    }
 
-                cfs.getWriteHandler().write(mutation.id(), upd, ctx, true);
+                    cfs.getWriteHandler().write(mutation.id(), upd, ctx, true);
+                }
             }
         }
 
-        MutationTrackingService.instance.finishWriting(mutation);
+        if (started)
+            MutationTrackingService.instance.finishWriting(mutation);
+
 
         if (future != null)
             future.trySuccess(null);
diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
index 5b19ebbf98..84932b2aa3 100644
--- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java
+++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java
@@ -55,6 +55,7 @@ public class MutationVerbHandler extends AbstractMutationVerbHandler<Mutation>
         if (ackTo != null)
         {
             logger.trace("Enqueuing response for direct acknowledgement of forwarded tracked mutation to coordinator {}", ackTo.coordinator);
+            response = Message.builder(response).withId(ackTo.id).build();
             MessagingService.instance().send(response, ackTo.coordinator);
         }
     }
diff --git a/src/java/org/apache/cassandra/dht/Splitter.java b/src/java/org/apache/cassandra/dht/Splitter.java
index 165a2ecf9b..6df33d6ad3 100644
--- a/src/java/org/apache/cassandra/dht/Splitter.java
+++ b/src/java/org/apache/cassandra/dht/Splitter.java
@@ -239,7 +239,7 @@ public abstract class Splitter extends AccordSplitter
      * @param parts the number of subranges
      * @return {@code parts} even subranges of {@code range}
      */
-    private Set<Range<Token>> split(Range<Token> range, int parts)
+    public Set<Range<Token>> split(Range<Token> range, int parts)
     {
         // the range might not have enough tokens to split
         BigInteger numTokens = tokensInRange(range);
diff --git a/src/java/org/apache/cassandra/journal/ActiveSegment.java b/src/java/org/apache/cassandra/journal/ActiveSegment.java
index 373b766267..d7c26c9104 100644
--- a/src/java/org/apache/cassandra/journal/ActiveSegment.java
+++ b/src/java/org/apache/cassandra/journal/ActiveSegment.java
@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
 
 import accord.utils.Invariants;
 
-import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.utils.Simulate;
 import org.apache.cassandra.utils.SyncUtil;
 import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -375,9 +374,9 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
      */
 
     @SuppressWarnings({ "resource", "RedundantSuppression" }) // op group will be closed by Allocation#write()
-    Allocation allocate(int entrySize)
+    Allocation allocate(int recordSize)
     {
-        int totalSize = totalEntrySize(entrySize);
+        int totalSize = EntrySerializer.totalEntrySize(keySupport, recordSize, descriptor.userVersion);
         OpOrder.Group opGroup = appendOrder.start();
         try
         {
@@ -396,14 +395,6 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
         }
     }
 
-    private int totalEntrySize(int recordSize)
-    {
-        return EntrySerializer.headerSize(keySupport, descriptor.userVersion)
-             + recordSize
-             + TypeSizes.INT_SIZE // CRC
-        ;
-    }
-
     // allocate bytes in the segment, or return -1 if not enough space
     private int allocateBytes(int size)
     {
@@ -528,7 +519,7 @@ public final class ActiveSegment<K, V> extends Segment<K, V>
 
         RecordPointer recordPointer()
         {
-            return new RecordPointer(descriptor.timestamp, start);
+            return new RecordPointer(descriptor.timestamp, start, length);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/journal/EntrySerializer.java b/src/java/org/apache/cassandra/journal/EntrySerializer.java
index 70d1e934d4..2c35b0619f 100644
--- a/src/java/org/apache/cassandra/journal/EntrySerializer.java
+++ b/src/java/org/apache/cassandra/journal/EntrySerializer.java
@@ -231,6 +231,16 @@ public final class EntrySerializer
              + TypeSizes.INT_SIZE;                    // CRC
     }
 
+    static <K> int overheadSize(KeySupport<K> keySupport, int userVersion)
+    {
+        return headerSize(keySupport, userVersion) + TypeSizes.INT_SIZE; // CRC
+    }
+
+    static <K> int totalEntrySize(KeySupport<K> keySupport, int recordSize, int userVersion)
+    {
+        return recordSize + overheadSize(keySupport, userVersion);
+    }
+
     public static final class EntryHolder<K>
     {
         public K key;
diff --git a/src/java/org/apache/cassandra/journal/Flusher.java b/src/java/org/apache/cassandra/journal/Flusher.java
index 70ec6d5593..bd1998eae6 100644
--- a/src/java/org/apache/cassandra/journal/Flusher.java
+++ b/src/java/org/apache/cassandra/journal/Flusher.java
@@ -73,7 +73,7 @@ final class Flusher<K, V>
     volatile long flushStartedAt;
     // the time of the earliest flush that has completed an fsync; all Allocations written before this time are durable
     volatile long fsyncFinishedFor = flushStartedAt;
-    volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0);
+    volatile RecordPointer fsyncFinishedForPosition = new RecordPointer(0, 0, 0);
 
     // a signal that writers can wait on to be notified of a completed flush in PERIODIC FlushMode
     private final WaitQueue fsyncComplete = newWaitQueue(); // TODO (expected): this is only used for testing, can we remove this?
@@ -214,7 +214,7 @@ final class Flusher<K, V>
                     fsyncedTo = fsyncTo.writtenToAtLeast();
                     fsyncTo.fsync();
                 }
-                fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, startedAt);
+                fsyncFinishedForPosition = new RecordPointer(fsyncTo.descriptor.timestamp, fsyncedTo, 0, startedAt);
                 fsyncFinishedFor = startedAt;
                 fsyncComplete.signalAll();
                 long finishedAt = clock.now();
@@ -383,7 +383,7 @@ final class Flusher<K, V>
 
         private void afterFSync(long startedAt, long segment, int position)
         {
-            fsyncFinishedForPosition = new RecordPointer(segment, position, startedAt);
+            fsyncFinishedForPosition = new RecordPointer(segment, position, 0, startedAt);
             fsyncFinishedFor = startedAt;
             callbacks.onFsync();
             fsyncComplete.signalAll();
@@ -481,7 +481,7 @@ final class Flusher<K, V>
         public RecordPointer flushAsync(ActiveSegment<K, V>.Allocation alloc)
         {
             written.incrementAndGet();
-            return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), clock.now());
+            return new RecordPointer(alloc.descriptor().timestamp, alloc.start(), 0, clock.now());
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/journal/Journal.java b/src/java/org/apache/cassandra/journal/Journal.java
index 29972bb20d..8d69f06afc 100644
--- a/src/java/org/apache/cassandra/journal/Journal.java
+++ b/src/java/org/apache/cassandra/journal/Journal.java
@@ -36,6 +36,8 @@ import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.zip.CRC32;
 
+import javax.annotation.Nullable;
+
 import com.codahale.metrics.Timer.Context;
 import com.google.common.annotations.VisibleForTesting;
 
@@ -482,6 +484,46 @@ public class Journal<K, V> implements Shutdownable
         return false;
     }
 
+    /**
+     * @param id user-provided record id
+     * @return record pointer of the last entry with the provided id, or null if not found
+     */
+    @Nullable
+    public RecordPointer lookUpLast(K id)
+    {
+        try (OpOrder.Group group = readOrder.start())
+        {
+            for (Segment<K, V> segment : segments.get().allSorted(false))
+            {
+                long[] offsets = segment.index().lookUp(id);
+                if (offsets.length != 0)
+                {
+                    long offsetAndSize = offsets[offsets.length - 1];
+                    int offset = Index.readOffset(offsetAndSize);
+                    int size = Index.readSize(offsetAndSize);
+                    return new RecordPointer(segment.descriptor.timestamp, offset, size);
+                }
+            }
+        }
+        return null;
+    }
+
+    public int sizeOfRecord(RecordPointer pointer)
+    {
+        Descriptor descriptor = segments.get().descriptor(pointer.segment);
+        Invariants.nonNull(descriptor);
+        return pointer.size - EntrySerializer.overheadSize(keySupport, descriptor.userVersion);
+    }
+
+    public boolean read(RecordPointer pointer, RecordConsumer<K> consumer)
+    {
+        try (OpOrder.Group group = readOrder.start())
+        {
+            Segment<K, V> segment = segments.get().get(pointer.segment);
+            return segment != null && segment.read(pointer, consumer);
+        }
+    }
+
     /**
      * Synchronously write a record to the journal.
      * <p/>
diff --git a/src/java/org/apache/cassandra/journal/RecordPointer.java b/src/java/org/apache/cassandra/journal/RecordPointer.java
index 1439e63c90..22d874712d 100644
--- a/src/java/org/apache/cassandra/journal/RecordPointer.java
+++ b/src/java/org/apache/cassandra/journal/RecordPointer.java
@@ -24,25 +24,27 @@ import com.google.common.primitives.Longs;
 // TODO: make this available in the accord table as an ID
 public class RecordPointer implements Comparable<RecordPointer>
 {
-    public final long segment; // unique segment id
-    public final int position; // record start position within the segment
+    public final long segment;   // unique segment id
+    public final int position;   // record start position within the segment
+    public final int size;       // full size of the record
     public final long writtenAt; // only set for periodic mode
 
-    public RecordPointer(long segment, int position)
+    public RecordPointer(long segment, int position, int size)
     {
-        this(segment, position, 0);
+        this(segment, position, size, 0);
     }
 
-    public RecordPointer(long segment, int position, long writtenAt)
+    public RecordPointer(long segment, int position, int size, long writtenAt)
     {
         this.segment = segment;
         this.position = position;
+        this.size = size;
         this.writtenAt = writtenAt;
     }
 
     public RecordPointer(RecordPointer pointer)
     {
-        this(pointer.segment, pointer.position, pointer.writtenAt);
+        this(pointer.segment, pointer.position, pointer.size, pointer.writtenAt);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/journal/Segment.java b/src/java/org/apache/cassandra/journal/Segment.java
index 7335bb2ca2..361f156f9a 100644
--- a/src/java/org/apache/cassandra/journal/Segment.java
+++ b/src/java/org/apache/cassandra/journal/Segment.java
@@ -109,6 +109,17 @@ public abstract class Segment<K, V> implements SelfRefCounted<Segment<K, V>>, Co
         return true;
     }
 
+    boolean read(RecordPointer pointer, RecordConsumer<K> consumer)
+    {
+        EntrySerializer.EntryHolder<K> into = new EntrySerializer.EntryHolder<>();
+        if (read(pointer.position, pointer.size, into))
+        {
+            consumer.accept(descriptor.timestamp, pointer.position, into.key, into.value, descriptor.userVersion);
+            return true;
+        }
+        return false;
+    }
+
     void readAll(K id, EntrySerializer.EntryHolder<K> into, RecordConsumer<K> onEntry)
     {
         long[] all = index().lookUpAll(id);
diff --git a/src/java/org/apache/cassandra/journal/Segments.java b/src/java/org/apache/cassandra/journal/Segments.java
index ea8dafd304..eadd5e17f2 100644
--- a/src/java/org/apache/cassandra/journal/Segments.java
+++ b/src/java/org/apache/cassandra/journal/Segments.java
@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.function.Predicate;
 
 import org.agrona.collections.Long2ObjectHashMap;
+import javax.annotation.Nullable;
 
 import accord.utils.Invariants;
 import accord.utils.SortedArrays.SortedArrayList;
@@ -104,15 +105,11 @@ public class Segments<K, V>
         return this.segments.values();
     }
 
-    public int count(Predicate<? super Segment<K, V>> predicate)
+    @Nullable
+    Descriptor descriptor(long timestamp)
     {
-        int count = 0;
-        for (Segment<K, V> segment : segments.values())
-        {
-            if (predicate.test(segment))
-                ++count;
-        }
-        return count;
+        Segment<K, V> segment = segments.get(timestamp);
+        return segment == null ? null : segment.descriptor;
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/locator/EndpointsForRange.java b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
index 6ac12707c2..c75f01f647 100644
--- a/src/java/org/apache/cassandra/locator/EndpointsForRange.java
+++ b/src/java/org/apache/cassandra/locator/EndpointsForRange.java
@@ -61,6 +61,11 @@ public class EndpointsForRange extends Endpoints<EndpointsForRange>
         return new EndpointsForToken(token, list, byEndpoint);
     }
 
+    public EndpointsForRange withRange(Range<Token> range)
+    {
+        return new EndpointsForRange(range, list, byEndpoint);
+    }
+
     @Override
     public EndpointsForRange snapshot()
     {
diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java
index 272266e0ef..f8ecfd58a6 100644
--- a/src/java/org/apache/cassandra/net/Message.java
+++ b/src/java/org/apache/cassandra/net/Message.java
@@ -19,11 +19,7 @@ package org.apache.cassandra.net;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
@@ -259,6 +255,11 @@ public class Message<T> implements ResponseContext
         return outWithParam(nextId(), verb, 0, payload, flag2.addTo(flag1.addTo(0)), null, null);
     }
 
+    public static <T> Message<T> outWithRequestTime(Verb verb, T payload, Dispatcher.RequestTime requestTime)
+    {
+        return outWithFlags(verb, payload, requestTime, Collections.emptyList());
+    }
+
     public static <T> Message<T> outWithFlags(Verb verb, T payload, Dispatcher.RequestTime requestTime, List<MessageFlag> flags)
     {
         assert !verb.isResponse();
diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java
index fa162b4cf4..79362bf5dc 100644
--- a/src/java/org/apache/cassandra/net/Verb.java
+++ b/src/java/org/apache/cassandra/net/Verb.java
@@ -77,8 +77,10 @@ import org.apache.cassandra.repair.messages.SyncRequest;
 import org.apache.cassandra.repair.messages.SyncResponse;
 import org.apache.cassandra.repair.messages.ValidationRequest;
 import org.apache.cassandra.repair.messages.ValidationResponse;
+import org.apache.cassandra.replication.BroadcastLogOffsets;
 import org.apache.cassandra.replication.ForwardedWrite;
-import org.apache.cassandra.replication.ShardReplicatedOffsets;
+import org.apache.cassandra.replication.PullMutationsRequest;
+import org.apache.cassandra.replication.PushMutationRequest;
 import org.apache.cassandra.schema.SchemaMutationsSerializer;
 import org.apache.cassandra.schema.SchemaPullVerbHandler;
 import org.apache.cassandra.schema.SchemaPushVerbHandler;
@@ -133,9 +135,7 @@ import org.apache.cassandra.service.paxos.cleanup.PaxosStartPrepareCleanup;
 import org.apache.cassandra.service.paxos.cleanup.PaxosUpdateLowBallot;
 import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler;
 import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileNotify;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileReceive;
-import org.apache.cassandra.service.reads.tracked.ReadReconcileSend;
+import org.apache.cassandra.service.reads.tracked.ReadReconcileAck;
 import org.apache.cassandra.service.reads.tracked.TrackedDataResponse;
 import org.apache.cassandra.service.reads.tracked.TrackedRead;
 import org.apache.cassandra.service.reads.tracked.TrackedSummaryResponse;
@@ -322,24 +322,25 @@ public enum Verb
     TCM_FETCH_PEER_LOG_RSP (818, P0, shortTimeout,    FETCH_METADATA,       MessageSerializers::logStateSerializer,             RESPONSE_HANDLER                                 ),
     TCM_FETCH_PEER_LOG_REQ (819, P0, rpcTimeout,      FETCH_METADATA,       () -> FetchPeerLog.serializer,                      () -> FetchPeerLog.Handler.instance,        TCM_FETCH_PEER_LOG_RSP ),
 
+    INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () -> NoPayload.serializer,             RESPONSE_HANDLER                                  ),
+    INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () -> DataMovement.serializer,          () -> DataMovementVerbHandler.instance, INITIATE_DATA_MOVEMENTS_RSP ),
+    DATA_MOVEMENT_EXECUTED_RSP  (816, P1, rpcTimeout, MISC, () -> NoPayload.serializer,             RESPONSE_HANDLER                                  ),
+    DATA_MOVEMENT_EXECUTED_REQ  (817, P1, rpcTimeout, MISC, () -> DataMovement.Status.serializer,   () -> DataMovements.instance,           DATA_MOVEMENT_EXECUTED_RSP  ),
+
     // tracked replication
-    READ_RECONCILE_SEND        (901, P0, rpcTimeout,   READ,             () -> ReadReconcileSend.serializer,          () -> ReadReconcileSend.verbHandler                                ),
-    READ_RECONCILE_RCV         (902, P0, rpcTimeout,   MUTATION,         () -> ReadReconcileReceive.serializer,       () -> ReadReconcileReceive.verbHandler                             ),
-    READ_RECONCILE_NOTIFY      (903, P0, rpcTimeout,   REQUEST_RESPONSE, () -> ReadReconcileNotify.serializer,        () -> ReadReconcileNotify.verbHandler                              ),
-    FORWARDING_WRITE           (904, P3, writeTimeout, MUTATION,         () -> ForwardedWrite.Request.serializer,     () -> ForwardedWrite.verbHandler),
-    BROADCAST_LOG_OFFSETS      (905, P1, rpcTimeout,   MISC,             () -> ShardReplicatedOffsets.serializer,     () -> ShardReplicatedOffsets.verbHandler),
-
-    TRACKED_PARTITION_READ_RSP (906, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER                                 ),
-    TRACKED_PARTITION_READ_REQ (907, P3, readTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_PARTITION_READ_RSP),
+    PULL_MUTATIONS_REQ         (901, P2, readTimeout,  READ,             () -> PullMutationsRequest.serializer,       () -> PullMutationsRequest.verbHandler                             ),
+    PUSH_MUTATION_REQ          (902, P3, writeTimeout, MUTATION,         () -> PushMutationRequest.serializer,        () -> PushMutationRequest.verbHandler,   MUTATION_RSP              ),
+    READ_RECONCILE_ACK         (903, P2, readTimeout,  REQUEST_RESPONSE, () -> ReadReconcileAck.serializer,           () -> ReadReconcileAck.verbHandler                                 ),
+    FORWARD_WRITE_REQ          (904, P3, writeTimeout, MUTATION,         () -> ForwardedWrite.serializer,             () -> ForwardedWrite.verbHandler                                   ),
+    BROADCAST_LOG_OFFSETS      (905, P1, rpcTimeout,   MISC,             () -> BroadcastLogOffsets.serializer,        () -> BroadcastLogOffsets.verbHandler                              ),
+
+    TRACKED_PARTITION_READ_RSP (906, P2, readTimeout,  REQUEST_RESPONSE, () -> TrackedDataResponse.serializer,        RESPONSE_HANDLER                                 ),
+    TRACKED_PARTITION_READ_REQ (907, P3, readTimeout,  READ,             () -> TrackedRead.DataRequest.serializer,    () -> TrackedRead.verbHandler, TRACKED_PARTITION_READ_RSP),
     TRACKED_RANGE_READ_RSP     (908, P2, rangeTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer,        RESPONSE_HANDLER                                 ),
     TRACKED_RANGE_READ_REQ     (909, P3, rangeTimeout, READ,             () -> TrackedRead.DataRequest.serializer,    () -> TrackedRead.verbHandler,           TRACKED_RANGE_READ_RSP    ),
-    TRACKED_SUMMARY_RSP        (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler                           ),
+    TRACKED_SUMMARY_RSP        (910, P2, readTimeout,  REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler                           ),
     TRACKED_SUMMARY_REQ        (911, P3, readTimeout,  READ,             () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler,           TRACKED_SUMMARY_RSP       ),
 
-    INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () -> NoPayload.serializer,             RESPONSE_HANDLER                                  ),
-    INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () -> DataMovement.serializer,          () -> DataMovementVerbHandler.instance, INITIATE_DATA_MOVEMENTS_RSP ),
-    DATA_MOVEMENT_EXECUTED_RSP  (816, P1, rpcTimeout, MISC, () -> NoPayload.serializer,             RESPONSE_HANDLER                                  ),
-    DATA_MOVEMENT_EXECUTED_REQ  (817, P1, rpcTimeout, MISC, () -> DataMovement.Status.serializer,   () -> DataMovements.instance,           DATA_MOVEMENT_EXECUTED_RSP  ),
 
     // accord
     ACCORD_SIMPLE_RSP               (119, P2, writeTimeout, IMMEDIATE,          () -> accordEmbedded(EnumSerializer.simpleReply),           AccordService::responseHandlerOrNoop                                           ),
@@ -418,8 +419,6 @@ public enum Verb
     @Deprecated(since = "4.0")
     INTERNAL_RSP           (23,  P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> ResponseVerbHandler.instance                             ),
 
-    // largest used ID: 116
-
     // CUSTOM VERBS
     UNUSED_CUSTOM_VERB     (CUSTOM,
                             0,   P1, rpcTimeout,      INTERNAL_RESPONSE, () -> null,                                 () -> null                                                     ),
diff --git a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
new file mode 100644
index 0000000000..54ded3766c
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.agrona.concurrent.ManyToOneConcurrentLinkedQueue;
+import org.apache.cassandra.concurrent.Interruptible;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.exceptions.RequestFailure;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessageFlag;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.net.RequestCallback;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.utils.concurrent.Semaphore;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SystemThreadTag.NON_DAEMON;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.Interrupts.SYNCHRONIZED;
+import static org.apache.cassandra.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static org.apache.cassandra.utils.concurrent.Semaphore.newSemaphore;
+
+// TODO (expected): handle temporarily down nodes
+public final class ActiveLogReconciler implements Shutdownable
+{
+    public enum Priority { HIGH, REGULAR }
+
+    // prioritised delivery of mutations that are needed by reads;
+    private final ManyToOneConcurrentLinkedQueue<Task> highPriorityTasks;
+
+    // regular write retries
+    private final ManyToOneConcurrentLinkedQueue<Task> regularPriorityTasks;
+
+    private final Interruptible executor;
+    private final Semaphore haveWork;
+
+    ActiveLogReconciler()
+    {
+        highPriorityTasks = new ManyToOneConcurrentLinkedQueue<>();
+        regularPriorityTasks = new ManyToOneConcurrentLinkedQueue<>();
+        haveWork = newSemaphore(1);
+        executor = executorFactory().infiniteLoop("Active-Log-Reconciler", new SendRunnable(), SAFE, NON_DAEMON, SYNCHRONIZED);
+    }
+
+    /**
+     * Schedule delivery of a mutation to the specified host
+     * TODO (expected): deduplicate via OutgoingMutations
+     */
+    void schedule(ShortMutationId mutationId, InetAddressAndPort toHost, Priority priority)
+    {
+        queue(priority).offer(new Task(mutationId, toHost));
+        haveWork.release(1);
+    }
+
+    /**
+     * Schedule delivery of mutations to the specified host
+     * TODO (expected): deduplicate via OutgoingMutations
+     */
+    void schedule(Offsets offsets, InetAddressAndPort toHost, Priority priority)
+    {
+        ManyToOneConcurrentLinkedQueue<Task> queue = queue(priority);
+        offsets.forEach(id -> queue.offer(new Task(id, toHost)));
+        haveWork.release(1);
+    }
+
+    private ManyToOneConcurrentLinkedQueue<Task> queue(Priority priority)
+    {
+        switch (priority)
+        {
+            case HIGH: return highPriorityTasks;
+            case REGULAR: return regularPriorityTasks;
+            default: throw new IllegalStateException();
+        }
+    }
+
+    private class SendRunnable implements Interruptible.Task
+    {
+        @Override
+        public void run(Interruptible.State state) throws InterruptedException
+        {
+            if (isPaused || isShutdown) return;
+
+            // TODO (expected): backoff, rate limits, per host and total
+            Task task;
+            while ((task = highPriorityTasks.poll()) != null)
+                task.send();
+            while ((task = regularPriorityTasks.poll()) != null)
+                task.send();
+
+            haveWork.acquire(1);
+        }
+    }
+
+    private static final class Task implements RequestCallback<NoPayload>
+    {
+        private final ShortMutationId mutationId;
+        private final InetAddressAndPort toHost;
+
+        Task(ShortMutationId mutationId, InetAddressAndPort toHost)
+        {
+            this.mutationId = mutationId;
+            this.toHost = toHost;
+        }
+
+        @Override
+        public boolean invokeOnFailure()
+        {
+            return true;
+        }
+
+        @Override
+        public void onResponse(Message<NoPayload> msg)
+        {
+            MutationTrackingService.instance.receivedWriteResponse(mutationId, toHost);
+        }
+
+        @Override
+        public void onFailure(InetAddressAndPort from, RequestFailure failureReason)
+        {
+            MutationTrackingService.instance.retryFailedWrite(mutationId, toHost, failureReason);
+        }
+
+        void send()
+        {
+            RecordPointer pointer = MutationJournal.instance.lookUp(mutationId);
+            Preconditions.checkNotNull(pointer, "Mutation %s not found in the journal", mutationId);
+
+            Message<PushMutationRequest> message =
+            Message.outWithFlag(Verb.PUSH_MUTATION_REQ,
+                                new PushMutationRequest.Referenced(mutationId, pointer),
+                                MessageFlag.CALL_BACK_ON_FAILURE);
+            MessagingService.instance().sendWithCallback(message, toHost, this);
+        }
+    }
+
+    private volatile boolean isShutdown = false;
+    private volatile boolean isPaused = false;
+
+    @Override
+    public boolean isTerminated()
+    {
+        return executor.isTerminated();
+    }
+
+    @Override
+    public void shutdown()
+    {
+        executor.shutdown();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return executor.shutdownNow();
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    {
+        return executor.awaitTermination(timeout, units);
+    }
+
+    public void shutdownBlocking() throws InterruptedException
+    {
+        isShutdown = true;
+        if (!executor.isTerminated())
+        {
+            executor.shutdown();
+            executor.awaitTermination(1, MINUTES);
+        }
+    }
+
+    @VisibleForTesting
+    void pauseForTesting()
+    {
+        isPaused = true;
+    }
+
+    @VisibleForTesting
+    void resumeForTesting()
+    {
+        isPaused = false;
+    }
+}
diff --git a/src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java b/src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
similarity index 82%
rename from src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java
rename to src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
index 267b63c9d2..b3fc118613 100644
--- a/src/java/org/apache/cassandra/replication/ShardReplicatedOffsets.java
+++ b/src/java/org/apache/cassandra/replication/BroadcastLogOffsets.java
@@ -34,15 +34,15 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.net.IVerbHandler;
 
-public class ShardReplicatedOffsets
+public class BroadcastLogOffsets
 {
-    private static final Logger logger = LoggerFactory.getLogger(ShardReplicatedOffsets.class);
+    private static final Logger logger = LoggerFactory.getLogger(BroadcastLogOffsets.class);
 
     private final String keyspace;
     private final Range<Token> range;
     private final List<Offsets.Immutable> replicatedOffsets;
 
-    public ShardReplicatedOffsets(String keyspace, Range<Token> range, List<Offsets.Immutable> offsets)
+    public BroadcastLogOffsets(String keyspace, Range<Token> range, List<Offsets.Immutable> offsets)
     {
         this.keyspace = keyspace;
         this.range = range;
@@ -60,8 +60,8 @@ public class ShardReplicatedOffsets
         return "ShardReplicatedOffsets{" + keyspace + ", " + range + ", " + replicatedOffsets + '}';
     }
 
-    public static final IVerbHandler<ShardReplicatedOffsets> verbHandler = message -> {
-        ShardReplicatedOffsets replicatedOffsets = message.payload;
+    public static final IVerbHandler<BroadcastLogOffsets> verbHandler = message -> {
+        BroadcastLogOffsets replicatedOffsets = message.payload;
         logger.trace("Received replicated offsets {} from {}", replicatedOffsets, message.from());
         MutationTrackingService.instance.updateReplicatedOffsets(replicatedOffsets.keyspace,
                                                                  replicatedOffsets.range,
@@ -69,10 +69,10 @@ public class ShardReplicatedOffsets
                                                                  message.from());
     };
 
-    public static final IVersionedSerializer<ShardReplicatedOffsets> serializer = new IVersionedSerializer<>()
+    public static final IVersionedSerializer<BroadcastLogOffsets> serializer = new IVersionedSerializer<>()
     {
         @Override
-        public void serialize(ShardReplicatedOffsets status, DataOutputPlus out, int version) throws IOException
+        public void serialize(BroadcastLogOffsets status, DataOutputPlus out, int version) throws IOException
         {
             out.writeUTF(status.keyspace);
             AbstractBounds.tokenSerializer.serialize(status.range, out, version);
@@ -82,7 +82,7 @@ public class ShardReplicatedOffsets
         }
 
         @Override
-        public ShardReplicatedOffsets deserialize(DataInputPlus in, int version) throws IOException
+        public BroadcastLogOffsets deserialize(DataInputPlus in, int version) throws IOException
         {
             String keyspace = in.readUTF();
             Range<Token> range = (Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, IPartitioner.global(), version);
@@ -90,11 +90,11 @@ public class ShardReplicatedOffsets
             List<Offsets.Immutable> replicatedOffsets = new ArrayList<>(count);
             for (int i = 0; i < count; ++i)
                 replicatedOffsets.add(Offsets.serializer.deserialize(in, version));
-            return new ShardReplicatedOffsets(keyspace, range, replicatedOffsets);
+            return new BroadcastLogOffsets(keyspace, range, replicatedOffsets);
         }
 
         @Override
-        public long serializedSize(ShardReplicatedOffsets replicatedOffsets, int version)
+        public long serializedSize(BroadcastLogOffsets replicatedOffsets, int version)
         {
             long size = 0;
             size += TypeSizes.sizeof(replicatedOffsets.keyspace);
diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLog.java b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
index 1470bf594d..13228256e2 100644
--- a/src/java/org/apache/cassandra/replication/CoordinatorLog.java
+++ b/src/java/org/apache/cassandra/replication/CoordinatorLog.java
@@ -28,6 +28,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Preconditions;
+
+import org.agrona.collections.IntArrayList;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -45,18 +47,21 @@ public abstract class CoordinatorLog
     protected final CoordinatorLogId logId;
     protected final Participants participants;
 
+    protected final UnreconciledMutations unreconciledMutations;
     protected final Offsets.Mutable[] witnessedOffsets;
     protected final Offsets.Mutable reconciledOffsets;
 
     protected final ReadWriteLock lock;
 
-    abstract UnreconciledMutations unreconciledMutations();
+    abstract void receivedWriteResponse(ShortMutationId mutationId, int fromHostId);
 
     CoordinatorLog(int localHostId, CoordinatorLogId logId, Participants participants)
     {
         this.localHostId = localHostId;
         this.logId = logId;
         this.participants = participants;
+
+        this.unreconciledMutations = new UnreconciledMutations();
         this.lock = new ReentrantReadWriteLock();
 
         Offsets.Mutable[] ids = new Offsets.Mutable[participants.size()];
@@ -73,34 +78,6 @@ public abstract class CoordinatorLog
                                         : new CoordinatorLogReplica(localHostId, id, participants);
     }
 
-    void receivedWriteResponse(MutationId mutationId, int onHostId)
-    {
-        Preconditions.checkArgument(!mutationId.isNone());
-        Preconditions.checkArgument(!Objects.equals(onHostId, ClusterMetadata.current().myNodeId().id()));
-        logger.trace("witnessed remote mutation {} from {}", mutationId, onHostId);
-        lock.writeLock().lock();
-        try
-        {
-            if (!get(onHostId).add(mutationId.offset()))
-                return; // already witnessed; very uncommon but possible path
-
-            if (!getLocal().contains(mutationId.offset()))
-                return; // local host hasn't witnessed yet -> no cleanup needed
-
-            if (remoteReplicasWitnessed(mutationId.offset()))
-            {
-                logger.trace("marking mutation {} as fully reconciled", mutationId);
-                // if all replicas have now witnessed the id, remove it from the index
-                unreconciledMutations().remove(mutationId.offset());
-                reconciledOffsets.add(mutationId.offset());
-            }
-        }
-        finally
-        {
-            lock.writeLock().unlock();
-        }
-    }
-
     void updateReplicatedOffsets(Offsets offsets, int onHostId)
     {
         lock.writeLock().lock();
@@ -114,7 +91,7 @@ public abstract class CoordinatorLog
                     if (othersWitnessed(offset, onHostId))
                     {
                         reconciledOffsets.add(offset);
-                        unreconciledMutations().remove(offset);
+                        unreconciledMutations.remove(offset);
                     }
                 }
             });
@@ -140,15 +117,16 @@ public abstract class CoordinatorLog
         }
     }
 
-    void startWriting(Mutation mutation)
+    boolean startWriting(Mutation mutation)
     {
         lock.writeLock().lock();
         try
         {
             if (getLocal().contains(mutation.id().offset()))
-                return; // already witnessed; shouldn't get to this path often (duplicate mutation)
+                return false; // already witnessed; shouldn't get to this path often (duplicate mutation)
 
-            unreconciledMutations().startWriting(mutation);
+            unreconciledMutations.startWriting(mutation);
+            return true;
         }
         finally
         {
@@ -164,15 +142,16 @@ public abstract class CoordinatorLog
         try
         {
             int offset = mutation.id().offset();
+            // we've raced with another write, no need to do anything else
             if (!getLocal().add(offset))
-                throw new IllegalStateException("finishWriting() called on a locally witnessed mutation " + mutation.id());
+                return;
 
-            unreconciledMutations().finishWriting(mutation);
+            unreconciledMutations.finishWriting(mutation);
 
             if (remoteReplicasWitnessed(offset))
             {
                 reconciledOffsets.add(offset);
-                unreconciledMutations().remove(offset);
+                unreconciledMutations.remove(offset);
             }
         }
         finally
@@ -192,7 +171,7 @@ public abstract class CoordinatorLog
         return true;
     }
 
-    private boolean remoteReplicasWitnessed(int offset)
+    protected boolean remoteReplicasWitnessed(int offset)
     {
         return othersWitnessed(offset, localHostId);
     }
@@ -207,7 +186,7 @@ public abstract class CoordinatorLog
         try
         {
             reconciledInto.addAll(reconciledOffsets);
-            return unreconciledMutations().collect(token, tableId, includePending, unreconciledInto);
+            return unreconciledMutations.collect(token, tableId, includePending, unreconciledInto);
         }
         finally
         {
@@ -225,7 +204,39 @@ public abstract class CoordinatorLog
         try
         {
             reconciledInto.addAll(reconciledOffsets);
-            return unreconciledMutations().collect(range, tableId, includePending, unreconciledInto);
+            return unreconciledMutations.collect(range, tableId, includePending, unreconciledInto);
+        }
+        finally
+        {
+            lock.readLock().unlock();
+        }
+    }
+
+    /**
+     * Collect the offsets in {@code remoteOffsets} that are missing from the local log.
+     */
+    void collectLocallyMissingMutations(Offsets remoteOffsets, Log2OffsetsMap.Mutable into)
+    {
+        lock.readLock().lock();
+        try
+        {
+            into.add(Offsets.Immutable.difference(remoteOffsets, getLocal()));
+        }
+        finally
+        {
+            lock.readLock().unlock();
+        }
+    }
+
+    void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+    {
+        lock.readLock().lock();
+        try
+        {
+            remoteNodeIds.forEachInt(remoteNodeId -> {
+                Offsets missing = Offsets.Immutable.difference(get(remoteNodeId), localOffsets);
+                if (!missing.isEmpty()) into.add(remoteNodeId, missing);
+            });
         }
         finally
         {
@@ -269,21 +280,42 @@ public abstract class CoordinatorLog
                '}';
     }
 
-    public static class CoordinatorLogPrimary extends CoordinatorLog
+    static class CoordinatorLogPrimary extends CoordinatorLog
     {
         private final AtomicLong sequenceId = new AtomicLong(-1);
-        private final UnreconciledMutationsReplica unreconciledMutations;
 
         CoordinatorLogPrimary(int localHostId, CoordinatorLogId logId, Participants participants)
         {
             super(localHostId, logId, participants);
-            unreconciledMutations = new UnreconciledMutationsReplica();
         }
 
         @Override
-        UnreconciledMutationsReplica unreconciledMutations()
+        void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
         {
-            return unreconciledMutations;
+            Preconditions.checkArgument(!mutationId.isNone());
+            Preconditions.checkArgument(!Objects.equals(fromHostId, ClusterMetadata.current().myNodeId().id()));
+            logger.trace("witnessed remote mutation {} from {}", mutationId, fromHostId);
+            lock.writeLock().lock();
+            try
+            {
+                if (!get(fromHostId).add(mutationId.offset()))
+                    return; // already witnessed; very uncommon but possible path
+
+                if (!getLocal().contains(mutationId.offset()))
+                    return; // local host hasn't witnessed yet -> no cleanup needed
+
+                if (remoteReplicasWitnessed(mutationId.offset()))
+                {
+                    logger.trace("marking mutation {} as fully reconciled", mutationId);
+                    // if all replicas have now witnessed the id, remove it from the index
+                    unreconciledMutations.remove(mutationId.offset());
+                    reconciledOffsets.add(mutationId.offset());
+                }
+            }
+            finally
+            {
+                lock.writeLock().unlock();
+            }
         }
 
         MutationId nextId()
@@ -309,20 +341,18 @@ public abstract class CoordinatorLog
         }
     }
 
-    public static class CoordinatorLogReplica extends CoordinatorLog
+    static class CoordinatorLogReplica extends CoordinatorLog
     {
-        private final UnreconciledMutationsReplica unreconciledMutations;
-
         CoordinatorLogReplica(int localHostId, CoordinatorLogId logId, Participants participants)
         {
             super(localHostId, logId, participants);
-            this.unreconciledMutations = new UnreconciledMutationsReplica();
         }
 
         @Override
-        UnreconciledMutationsReplica unreconciledMutations()
+        void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
         {
-            return unreconciledMutations;
+            // no-op
         }
     }
+
 }
diff --git a/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java b/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java
new file mode 100644
index 0000000000..552f62e105
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/ExpiredStatePurger.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
+import org.apache.cassandra.concurrent.Shutdownable;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.Clock;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
+
+/**
+ * Since most of our state machines don't use 2 way callbacks, expirations are handled here.
+ * </p>
+ * TODO (expected): switch to using accord.impl.RequestCallbacks instead of walking the maps
+ */
+public class ExpiredStatePurger implements Shutdownable
+{
+    private static final Logger logger = LoggerFactory.getLogger(ExpiredStatePurger.class);
+    public static final ExpiredStatePurger instance = new ExpiredStatePurger();
+
+    private final ScheduledExecutorPlus executor = executorFactory().scheduled("Expired-State-Purger", NORMAL);
+    private final CopyOnWriteArrayList<Expireable> expireables = new CopyOnWriteArrayList<>();
+
+    public interface Expireable
+    {
+        int expire(long nanoTime);
+    }
+
+    public ExpiredStatePurger()
+    {
+        long expirationInterval = defaultExpirationInterval();
+        executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS);
+    }
+
+    public void register(Expireable expireable)
+    {
+        expireables.add(expireable);
+    }
+
+    private void expire()
+    {
+        long nanoTime = Clock.Global.nanoTime();
+        for (Expireable expireable : expireables)
+        {
+            int n = expireable.expire(nanoTime);
+            if (n > 0) logger.trace("Expired {} {} entries", n, expireable);
+        }
+    }
+
+    private long defaultExpirationInterval()
+    {
+        return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2;
+    }
+
+    @Override
+    public void shutdown()
+    {
+        executor.shutdown();
+    }
+
+    @Override
+    public boolean isTerminated()
+    {
+        return executor.isTerminated();
+    }
+
+    @Override
+    public Object shutdownNow()
+    {
+        return executor.shutdownNow();
+    }
+
+    public void shutdownBlocking() throws InterruptedException
+    {
+        if (executor == null || executor.isTerminated())
+            return;
+
+        executor.shutdown();
+        executor.awaitTermination(1, MINUTES);
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
+    {
+        return executor.awaitTermination(timeout, units);
+    }
+}
diff --git a/src/java/org/apache/cassandra/replication/ForwardedWrite.java b/src/java/org/apache/cassandra/replication/ForwardedWrite.java
index e12e020b90..c6014257e8 100644
--- a/src/java/org/apache/cassandra/replication/ForwardedWrite.java
+++ b/src/java/org/apache/cassandra/replication/ForwardedWrite.java
@@ -15,7 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.replication;
 
 import java.io.IOException;
@@ -31,6 +30,7 @@ import com.google.common.collect.Sets;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.exceptions.RequestFailure;
 import org.apache.cassandra.locator.*;
+import org.apache.cassandra.tracing.Tracing;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,6 +55,8 @@ import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.net.Verb.MUTATION_REQ;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
 
 /**
  * For a forwarded write there are 2 nodes involved in coordination, a coordinator and a leader. The coordinator is the
@@ -65,91 +67,7 @@ public class ForwardedWrite
 {
     private static final Logger logger = LoggerFactory.getLogger(ForwardedWrite.class);
 
-    public interface Request
-    {
-        enum Kind
-        {
-            MUTATION(0);
-
-            private final byte id;
-
-            Kind(int id)
-            {
-                this.id = (byte) id;
-            }
-
-            IVersionedSerializer<Request> serializer()
-            {
-                switch (this)
-                {
-                    case MUTATION:
-                        return MutationRequest.serializer;
-                    default:
-                        throw new IllegalStateException("Unhandled kind " + this);
-                }
-            }
-
-            static final IVersionedSerializer<Kind> serializer = new IVersionedSerializer<Request.Kind>()
-            {
-                @Override
-                public void serialize(Kind kind, DataOutputPlus out, int version) throws IOException
-                {
-                    out.writeByte(kind.id);
-
-                }
-
-                @Override
-                public Kind deserialize(DataInputPlus in, int version) throws IOException
-                {
-                    byte id = in.readByte();
-                    switch (id)
-                    {
-                        case 0:
-                            return MUTATION;
-                        default:
-                            throw new IllegalStateException("Unknown kind: " + id);
-                    }
-                }
-
-                @Override
-                public long serializedSize(Kind kind, int version)
-                {
-                    return TypeSizes.BYTE_SIZE;
-                }
-            };
-        }
-
-        Kind kind();
-        DecoratedKey key();
-        void applyLocallyAndForwardToReplicas(CoordinatorAckInfo ackTo);
-
-        IVersionedSerializer<Request> serializer = new IVersionedSerializer<>()
-        {
-            @Override
-            public void serialize(Request request, DataOutputPlus out, int version) throws IOException
-            {
-                Kind.serializer.serialize(request.kind(), out, version);
-                request.kind().serializer().serialize(request, out, version);
-            }
-
-            @Override
-            public Request deserialize(DataInputPlus in, int version) throws IOException
-            {
-                Kind kind = Kind.serializer.deserialize(in, version);
-                return kind.serializer().deserialize(in, version);
-            }
-
-            @Override
-            public long serializedSize(Request request, int version)
-            {
-                long size = Kind.serializer.serializedSize(request.kind(), version);
-                size += request.kind().serializer().serializedSize(request, version);
-                return size;
-            }
-        };
-    }
-
-    public static class MutationRequest implements Request
+    public static class MutationRequest
     {
         private final Mutation mutation;
         private final Set<NodeId> recipients;
@@ -175,19 +93,11 @@ public class ForwardedWrite
             this.recipients = recipients;
         }
 
-        @Override
-        public Kind kind()
-        {
-            return Kind.MUTATION;
-        }
-
-        @Override
         public DecoratedKey key()
         {
             return mutation.key();
         }
 
-        @Override
         public void applyLocallyAndForwardToReplicas(CoordinatorAckInfo ackTo)
         {
             Preconditions.checkState(ackTo != null);
@@ -197,7 +107,7 @@ public class ForwardedWrite
 
             MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token);
             // Do not wait for handler completion, since the coordinator is already waiting and we don't want to block the stage
-            LeaderCallback handler = new LeaderCallback(keyspaceName, mutation.key().getToken(), id, ackTo);
+            LeaderCallback handler = new LeaderCallback(id, ackTo);
             applyLocallyAndForwardToReplicas(mutation.withMutationId(id), recipients, handler, ackTo);
         }
 
@@ -248,7 +158,6 @@ public class ForwardedWrite
                                      .withRequestTime(handler.getRequestTime())
                                      .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
                                      .withParam(ParamType.COORDINATOR_ACK_INFO, ackTo)
-                                     .withId(ackTo.id)
                                      .build();
 
                 Replica replica = replicas.get(recipient);
@@ -286,41 +195,6 @@ public class ForwardedWrite
                     TrackedWriteRequest.sendMessagesToRemoteDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcReplicas), handler, ackTo);
             }
         }
-
-        public static final IVersionedSerializer<Request> serializer = new IVersionedSerializer<>()
-        {
-            @Override
-            public void serialize(Request r, DataOutputPlus out, int version) throws IOException
-            {
-                MutationRequest request = (MutationRequest) r;
-                Mutation.serializer.serialize(request.mutation, out, version);
-                out.writeInt(request.recipients.size());
-                for (NodeId recipient : request.recipients)
-                    NodeId.messagingSerializer.serialize(recipient, out, version);
-            }
-
-            @Override
-            public Request deserialize(DataInputPlus in, int version) throws IOException
-            {
-                Mutation mutation = Mutation.serializer.deserialize(in, version);
-                int numRecipients = in.readInt();
-                Set<NodeId> recipients = Sets.newHashSetWithExpectedSize(numRecipients);
-                for (int i = 0; i < numRecipients; i++)
-                    recipients.add(NodeId.messagingSerializer.deserialize(in, version));
-                return new MutationRequest(mutation, recipients);
-            }
-
-            @Override
-            public long serializedSize(Request r, int version)
-            {
-                MutationRequest request = (MutationRequest) r;
-                long size = Mutation.serializer.serializedSize(request.mutation, version);
-                size += TypeSizes.INT_SIZE;
-                for (NodeId recipient : request.recipients)
-                    size += NodeId.messagingSerializer.serializedSize(recipient, version);
-                return size;
-            }
-        };
     }
 
     public static AbstractWriteResponseHandler<Object> forwardMutation(Mutation mutation, ReplicaPlan.ForWrite plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime)
@@ -331,25 +205,35 @@ public class ForwardedWrite
         Token token = mutation.key().getToken();
         Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
         EndpointsForRange endpoints = cm.placements.get(keyspace.getMetadata().params.replication).writes.forRange(token).get();
-        if (logger.isTraceEnabled())
-            logger.trace("Finding best leader from replicas {}", endpoints);
+        logger.trace("Finding best leader from replicas {}", endpoints);
 
         // TODO: Should match ReplicaPlans.findCounterLeaderReplica, including DC-local priority, current health, severity, etc.
-        Replica leader = proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints).get(0);
+        Replica leader = null;
+        for (Replica replica : proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints))
+        {
+            if (plan.isAlive(replica))
+                leader = replica;
+        }
+        Preconditions.checkState(leader != null, "Could not find leader for %s", mutation);
 
         // create callback and forward to leader
-        if (logger.isTraceEnabled())
-            logger.trace("Selected {} as leader for mutation with key {}", leader.endpoint(), mutation.key());
+        logger.trace("Selected {} as leader for mutation with key {}", leader.endpoint(), mutation.key());
 
         AbstractWriteResponseHandler<Object> handler = strategy.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime);
 
         // Add callbacks for replicas to respond directly to coordinator
-        Message<Request> toLeader = Message.out(Verb.FORWARDING_WRITE, new MutationRequest(mutation, plan));
+        Message<MutationRequest> toLeader = Message.outWithRequestTime(Verb.FORWARD_WRITE_REQ, new MutationRequest(mutation, plan), requestTime);
         for (Replica endpoint : endpoints)
         {
-            if (logger.isTraceEnabled())
+            if (plan.isAlive(endpoint))
+            {
                 logger.trace("Adding forwarding callback for response from {} id {}", endpoint, toLeader.id());
-            MessagingService.instance().callbacks.addWithExpiration(handler, toLeader, endpoint);
+                MessagingService.instance().callbacks.addWithExpiration(handler, toLeader, endpoint);
+            }
+            else
+            {
+                handler.expired();
+            }
         }
 
         MessagingService.instance().send(toLeader, leader.endpoint());
@@ -357,26 +241,60 @@ public class ForwardedWrite
         return handler;
     }
 
-    public static final IVerbHandler<Request> verbHandler = new IVerbHandler<>()
+    public static final IVersionedSerializer<MutationRequest> serializer = new IVersionedSerializer<>()
     {
         @Override
-        public void doVerb(Message<Request> incoming)
+        public void serialize(MutationRequest request, DataOutputPlus out, int version) throws IOException
         {
-            if (logger.isTraceEnabled())
-                logger.trace("Received incoming ForwardedWriteRequest {} id {}", incoming, incoming.id());
-            CoordinatorAckInfo ackTo = CoordinatorAckInfo.toCoordinator(incoming.from(), incoming.id());
-            Request request = incoming.payload;
+            Mutation.serializer.serialize(request.mutation, out, version);
+            out.writeInt(request.recipients.size());
+            for (NodeId recipient : request.recipients)
+                NodeId.messagingSerializer.serialize(recipient, out, version);
+        }
 
-            // Once we support epoch changes, check epoch from coordinator here, after potential queueing on the Stage
-            try
-            {
-                request.applyLocallyAndForwardToReplicas(ackTo);
-            }
-            catch (Exception e)
-            {
-                logger.error("Exception while executing forwarded write with key {} on leader", request.key(), e);
-                MessagingService.instance().respondWithFailure(RequestFailureReason.UNKNOWN, incoming);
-            }
+        @Override
+        public MutationRequest deserialize(DataInputPlus in, int version) throws IOException
+        {
+            Mutation mutation = Mutation.serializer.deserialize(in, version);
+            int numRecipients = in.readInt();
+            Set<NodeId> recipients = Sets.newHashSetWithExpectedSize(numRecipients);
+            for (int i = 0; i < numRecipients; i++)
+                recipients.add(NodeId.messagingSerializer.deserialize(in, version));
+            return new MutationRequest(mutation, recipients);
+        }
+
+        @Override
+        public long serializedSize(MutationRequest request, int version)
+        {
+            long size = Mutation.serializer.serializedSize(request.mutation, version);
+            size += TypeSizes.INT_SIZE;
+            for (NodeId recipient : request.recipients)
+                size += NodeId.messagingSerializer.serializedSize(recipient, version);
+            return size;
+        }
+    };
+
+    public static final IVerbHandler<MutationRequest> verbHandler = incoming ->
+    {
+        if (approxTime.now() > incoming.expiresAtNanos())
+        {
+            Tracing.trace("Discarding mutation from {} (timed out)", incoming.from());
+            MessagingService.instance().metrics.recordDroppedMessage(incoming, incoming.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+            return;
+        }
+        logger.trace("Received incoming ForwardedWriteRequest {} id {}", incoming, incoming.id());
+        CoordinatorAckInfo ackTo = CoordinatorAckInfo.toCoordinator(incoming.from(), incoming.id());
+        MutationRequest request = incoming.payload;
+
+        // Once we support epoch changes, check epoch from coordinator here, after potential queueing on the Stage
+        try
+        {
+            request.applyLocallyAndForwardToReplicas(ackTo);
+        }
+        catch (Exception e)
+        {
+            logger.error("Exception while executing forwarded write with key {} on leader", request.key(), e);
+            MessagingService.instance().respondWithFailure(RequestFailureReason.UNKNOWN, incoming);
         }
     };
 
@@ -384,16 +302,12 @@ public class ForwardedWrite
     // See org.apache.cassandra.service.TrackedWriteResponseHandler.onResponse, this class should probably merge with that one
     public static class LeaderCallback implements RequestCallback<NoPayload>
     {
-        private final String keyspace;
-        private final Token token;
         private final MutationId id;
         private final CoordinatorAckInfo ackTo;
         private final Dispatcher.RequestTime requestTime = Dispatcher.RequestTime.forImmediateExecution();
 
-        public LeaderCallback(String keyspace, Token token, MutationId id, CoordinatorAckInfo ackTo)
+        public LeaderCallback(MutationId id, CoordinatorAckInfo ackTo)
         {
-            this.keyspace = keyspace;
-            this.token = token;
             this.id = id;
             this.ackTo = ackTo;
         }
@@ -403,7 +317,7 @@ public class ForwardedWrite
         {
             // Local mutations are witnessed from Keyspace.applyInternalTracked
             if (msg != null)
-                MutationTrackingService.instance.receivedWriteResponse(keyspace, token, id, msg.from());
+                MutationTrackingService.instance.receivedWriteResponse(id, msg.from());
 
             // Local write needs to be ack'd to coordinator
             if (msg == null && ackTo != null)
diff --git a/src/java/org/apache/cassandra/replication/IncomingMutations.java b/src/java/org/apache/cassandra/replication/IncomingMutations.java
new file mode 100644
index 0000000000..d67e152c92
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/IncomingMutations.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.Clock;
+import org.jctools.maps.NonBlockingHashMap;
+
+/**
+ * Listeners for incoming mutations requested by e.g. {@link org.apache.cassandra.service.reads.tracked.TrackedLocalReads}.
+ * </p>
+ * The mutations being requested may not necessarily have a corresponding {@link CoordinatorLog}
+ * instance just yet - e.g. it's a mutation from a fresh coordinator log that we are requesting,
+ * and we were only made aware of it via missing offsets in read reconciliation.
+ * </p>
+ * For the above reason it's not folded into an instance {@link CoordinatorLog} but instead lives
+ * inside {@link MutationTrackingService}.
+ */
+public class IncomingMutations implements ExpiredStatePurger.Expireable
+{
+    private static final Logger logger = LoggerFactory.getLogger(IncomingMutations.class);
+
+    private final NonBlockingHashMap<ShortMutationId, Listeners> listenersMap = new NonBlockingHashMap<>();
+
+    /**
+     * Register to be notified to an incoming mutation.
+     * @return true if this is the first active listener added for this id
+     */
+    public boolean subscribe(ShortMutationId mutationId, Callback callback)
+    {
+        Listeners prev, next;
+        do
+        {
+            prev = listenersMap.get(mutationId);
+            next = Listeners.addListener(prev, callback);
+        }
+        while (listenersMap.putIfMatchAllowNull(mutationId, next, prev) != prev);
+        return prev == null;
+    }
+
+    public void invokeListeners(ShortMutationId mutationId)
+    {
+        Listeners listeners = listenersMap.remove(mutationId);
+        if (listeners != null) listeners.invokeListeners(mutationId);
+    }
+
+    @Override
+    public int expire(long nanoTime)
+    {
+        int n = 0;
+        for (Map.Entry<ShortMutationId, Listeners> entry : listenersMap.entrySet())
+        {
+            ShortMutationId id = entry.getKey();
+            Listeners listeners = entry.getValue();
+            if (listeners.isExpired(nanoTime) && listenersMap.remove(id, listeners))
+            {
+                listeners.expireListeners(id);
+                n++;
+            }
+        }
+        return n;
+    }
+
+    private static final class Listeners
+    {
+        private final long createdAt;
+        private final Callback[] callbacks;
+
+        Listeners(Callback callback)
+        {
+            this(new Callback[] { callback }, Clock.Global.nanoTime());
+        }
+
+        Listeners(Callback[] callbacks, long createdAt)
+        {
+            this.callbacks = callbacks;
+            this.createdAt = createdAt;
+        }
+
+        Listeners addListener(Callback callback)
+        {
+            Callback[] newCallbacks = new Callback[callbacks.length + 1];
+            System.arraycopy(callbacks, 0, newCallbacks, 0, callbacks.length);
+            newCallbacks[callbacks.length] = callback;
+            return new Listeners(newCallbacks, createdAt);
+        }
+
+        static Listeners addListener(Listeners listeners, Callback callback)
+        {
+            return listeners == null ? new Listeners(callback) : listeners.addListener(callback);
+        }
+
+        void invokeListeners(ShortMutationId mutationId)
+        {
+            for (Callback callback : callbacks)
+            {
+                try
+                {
+                    callback.onSuccess(mutationId);
+                }
+                catch (Throwable e)
+                {
+                    logger.error("Caught an error while processing onSuccess() callback for {}: {}", e, mutationId);
+                }
+            }
+        }
+
+        void expireListeners(ShortMutationId mutationId)
+        {
+            for (Callback callback : callbacks)
+            {
+                try
+                {
+                    callback.onTimeout(mutationId);
+                }
+                catch (Throwable e)
+                {
+                    logger.error("Caught an error while processing onTimeout() callback for {}: {}", e, mutationId);
+                }
+            }
+        }
+
+        boolean isExpired(long nanoTime)
+        {
+            return createdAt + DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.NANOSECONDS) < nanoTime;
+        }
+    }
+
+    public interface Callback
+    {
+        void onSuccess(ShortMutationId mutationId);
+
+        default void onTimeout(ShortMutationId mutationId)
+        {
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
index efcdd195ec..f7b03b15c5 100644
--- a/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
+++ b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java
@@ -19,6 +19,7 @@
 package org.apache.cassandra.replication;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Iterator;
 
 import com.google.common.base.Preconditions;
@@ -29,31 +30,34 @@ import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary;
 
 public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<ShortMutationId>
 {
-    abstract Long2ObjectHashMap<T> offsetMap();
+    abstract Long2ObjectHashMap<T> asMap();
 
     @Override
     public Iterator<ShortMutationId> iterator()
     {
-        return Iterables.concat(offsetMap().values()).iterator();
+        return Iterables.concat(asMap().values()).iterator();
+    }
+
+    public Collection<T> offsets()
+    {
+        return asMap().values();
     }
 
     public int idCount()
     {
         int count = 0;
-        for (T offsets : offsetMap().values())
+        for (T offsets : offsets())
             count += offsets.offsetCount();
         return count;
     }
 
     public boolean isEmpty()
     {
-        for (T offsets : offsetMap().values())
-            if (!offsets.isEmpty())
-                return false;
-        return true;
+        return Iterables.all(offsets(), Offsets::isEmpty);
     }
 
     private static abstract class AbstractMutable<T extends Offsets.AbstractMutable<T>> extends Log2OffsetsMap<T>
@@ -61,13 +65,15 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
         protected final Long2ObjectHashMap<T> offsetMap = new Long2ObjectHashMap<>();
 
         @Override
-        Long2ObjectHashMap<T> offsetMap()
+        Long2ObjectHashMap<T> asMap()
         {
             return offsetMap;
         }
 
         protected abstract T createOrNull(Offsets.RangeIterator iterator);
+
         protected abstract T create(CoordinatorLogId logId);
+
         protected abstract T copy(Offsets offsets);
 
         protected T create(long logId)
@@ -98,10 +104,22 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
 
         public void addAll(Log2OffsetsMap<?> that)
         {
-            for (Offsets offsets : that.offsetMap().values())
+            for (Offsets offsets : that.asMap().values())
                 add(offsets);
         }
 
+        public void addAll(MutationSummary summary)
+        {
+            for (int i = 0, size = summary.size(); i < size; i++)
+                addAll(summary.get(i));
+        }
+
+        public void addAll(CoordinatorSummary summary)
+        {
+            add(summary.reconciled);
+            add(summary.unreconciled);
+        }
+
         public void remove(Offsets offsets)
         {
             T existing = offsetMap.get(offsets.logId().asLong());
@@ -114,6 +132,7 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
                 return;
             }
 
+            // TODO (expected): ? shouldn't we updating Offsets in-place here? where did Offsets.remove() go?
             T next = createOrNull(Offsets.difference(existing.rangeIterator(), offsets.rangeIterator()));
             if (next == null)
                 offsetMap.remove(offsets.logId().asLong());
@@ -121,11 +140,41 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
                 offsetMap.put(offsets.logId().asLong(), next);
         }
 
+        public void remove(ShortMutationId id)
+        {
+            T existing = offsetMap.get(id.logId());
+            if (existing == null)
+                return;
+
+            existing.remove(id.offset());
+
+            if (existing.isEmpty())
+                offsetMap.remove(id.logId());
+        }
+
         public void removeAll(Log2OffsetsMap<?> that)
         {
-            for (Offsets offsets : that.offsetMap().values())
+            for (Offsets offsets : that.asMap().values())
                 remove(offsets);
         }
+
+        public void removeAll(Iterable<ShortMutationId> ids)
+        {
+            for (ShortMutationId id : ids)
+                remove(id);
+        }
+
+        public void removeAll(MutationSummary summary)
+        {
+            for (int i = 0, size = summary.size(); i < size; i++)
+                removeAll(summary.get(i));
+        }
+
+        public void removeAll(CoordinatorSummary summary)
+        {
+            remove(summary.reconciled);
+            remove(summary.unreconciled);
+        }
     }
 
     public static class Mutable extends AbstractMutable<Offsets.Mutable>
@@ -159,7 +208,7 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
         }
 
         @Override
-        Long2ObjectHashMap<Offsets.Immutable> offsetMap()
+        Long2ObjectHashMap<Offsets.Immutable> asMap()
         {
             return offsetMap;
         }
@@ -192,13 +241,13 @@ public abstract class Log2OffsetsMap<T extends Offsets> implements Iterable<Shor
             }
         }
 
-        public static final IVersionedSerializer<Log2OffsetsMap.Immutable> serializer = new IVersionedSerializer<Log2OffsetsMap.Immutable>()
+        public static final IVersionedSerializer<Log2OffsetsMap.Immutable> serializer = new IVersionedSerializer<>()
         {
             @Override
             public void serialize(Log2OffsetsMap.Immutable mo, DataOutputPlus out, int version) throws IOException
             {
                 out.writeInt(mo.offsetMap.size());
-                for (Offsets.Immutable offsets : mo.offsetMap().values())
+                for (Offsets.Immutable offsets : mo.asMap().values())
                     Offsets.serializer.serialize(offsets, out, version);
             }
 
diff --git a/src/java/org/apache/cassandra/replication/MutationJournal.java b/src/java/org/apache/cassandra/replication/MutationJournal.java
index 31730e98c6..bb0a93cd6a 100644
--- a/src/java/org/apache/cassandra/replication/MutationJournal.java
+++ b/src/java/org/apache/cassandra/replication/MutationJournal.java
@@ -75,11 +75,30 @@ public class MutationJournal
         return journal.readLast(id);
     }
 
-    public boolean read(ShortMutationId id, RecordConsumer<ShortMutationId> consumer)
+    boolean read(ShortMutationId id, RecordConsumer<ShortMutationId> consumer)
     {
         return journal.readLast(id, consumer);
     }
 
+    /**
+     * @return record pointer of the last mutation with the provided id, or null if not found
+     */
+    @Nullable
+    RecordPointer lookUp(ShortMutationId id)
+    {
+        return journal.lookUpLast(id);
+    }
+
+    int sizeOfRecord(RecordPointer pointer)
+    {
+        return journal.sizeOfRecord(pointer);
+    }
+
+    boolean read(RecordPointer pointer, RecordConsumer<ShortMutationId> consumer)
+    {
+        return journal.read(pointer, consumer);
+    }
+
     public void readAll(Iterable<ShortMutationId> ids, Collection<Mutation> into)
     {
         for (ShortMutationId id : ids)
diff --git a/src/java/org/apache/cassandra/replication/MutationSummary.java b/src/java/org/apache/cassandra/replication/MutationSummary.java
index c786b295fe..f3bc6051d2 100644
--- a/src/java/org/apache/cassandra/replication/MutationSummary.java
+++ b/src/java/org/apache/cassandra/replication/MutationSummary.java
@@ -29,6 +29,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.AbstractIterator;
 
 public class MutationSummary
 {
@@ -324,6 +325,22 @@ public class MutationSummary
         }
     }
 
+    public Iterator<Offsets> onlyUnreconciled()
+    {
+        return new AbstractIterator<>()
+        {
+            int i = 0;
+
+            @Override
+            protected Offsets computeNext()
+            {
+                if (i >= summaries.size()) return endOfData();
+                Offsets offsets = summaries.get(i++).unreconciled;
+                return offsets.isEmpty() ? computeNext() : offsets;
+            }
+        };
+    }
+
     public static final IVersionedSerializer<MutationSummary> serializer = new IVersionedSerializer<>()
     {
         @Override
diff --git a/src/java/org/apache/cassandra/replication/MutationTrackingService.java b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
index c22ac87007..7ec6758707 100644
--- a/src/java/org/apache/cassandra/replication/MutationTrackingService.java
+++ b/src/java/org/apache/cassandra/replication/MutationTrackingService.java
@@ -17,19 +17,28 @@
  */
 package org.apache.cassandra.replication;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.IntSupplier;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 import org.agrona.collections.IntArrayList;
+import org.agrona.collections.IntHashSet;
 import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
 import org.apache.cassandra.concurrent.Shutdownable;
 import org.apache.cassandra.db.DecoratedKey;
@@ -37,7 +46,9 @@ import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestFailure;
 import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.Message;
@@ -48,11 +59,14 @@ import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableId;
 import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
 import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ownership.ReplicaGroups;
+import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
 import org.apache.cassandra.utils.FBUtilities;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
 
@@ -60,12 +74,25 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics
 // TODO (expected): handle topology changes
 public class MutationTrackingService
 {
+    /**
+     * Split ranges into this many shards.
+     *
+     * TODO (expected): ability to rebalance / change this constant
+     */
+    private static final int SHARD_MULTIPLIER = 8;
+
     private static final Logger logger = LoggerFactory.getLogger(MutationTrackingService.class);
     public static final MutationTrackingService instance = new MutationTrackingService();
 
     private final TrackedLocalReads localReads = new TrackedLocalReads();
-    private final ReplicatedOffsetsBroadcaster broadcaster = new ReplicatedOffsetsBroadcaster();
-    private final ConcurrentHashMap<String, KeyspaceShards> shards = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, KeyspaceShards> keyspaceShards = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<CoordinatorLogId, Shard> log2ShardMap = new ConcurrentHashMap<>();
+
+    private final ReplicatedOffsetsBroadcaster offsetsBroadcaster = new ReplicatedOffsetsBroadcaster();
+    private final ActiveLogReconciler activeReconciler = new ActiveLogReconciler();
+
+    private final IncomingMutations incomingMutations = new IncomingMutations();
+    private final OutgoingMutations outgoingMutations = new OutgoingMutations();
 
     private volatile boolean started = false;
 
@@ -81,9 +108,11 @@ public class MutationTrackingService
 
         for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces())
             if (keyspace.useMutationTracking())
-                shards.put(keyspace.name, KeyspaceShards.make(keyspace, metadata, this::nextHostLogId));
+                keyspaceShards.put(keyspace.name, KeyspaceShards.make(keyspace, metadata, this::nextHostLogId, this::onNewLog));
 
-        broadcaster.start();
+        offsetsBroadcaster.start();
+
+        ExpiredStatePurger.instance.register(incomingMutations);
 
         started = true;
     }
@@ -95,9 +124,10 @@ public class MutationTrackingService
 
     public void shutdownBlocking() throws InterruptedException
     {
-        localReads.shutdownBlocking();
-        broadcaster.shutdown();
-        broadcaster.awaitTermination(1, TimeUnit.MINUTES);
+        offsetsBroadcaster.shutdown();
+        offsetsBroadcaster.awaitTermination(1, TimeUnit.MINUTES);
+        activeReconciler.shutdownBlocking();
+        ExpiredStatePurger.instance.shutdownBlocking();
     }
 
     public TrackedLocalReads localReads()
@@ -107,42 +137,69 @@ public class MutationTrackingService
 
     public MutationId nextMutationId(String keyspace, Token token)
     {
-        MutationId id = getOrCreate(keyspace).nextMutationId(token);
+        MutationId id = getOrCreateShards(keyspace).nextMutationId(token);
         logger.trace("Created new mutation id {}", id);
         return id;
     }
 
-    public void receivedWriteResponse(String keyspace, Token token, MutationId mutationId, InetAddressAndPort onHost)
+    public void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+    {
+        Preconditions.checkArgument(!mutation.id().isNone());
+//        outgoingMutations.sentWriteRequest(mutation, toHostIds);
+    }
+
+    public void receivedWriteResponse(ShortMutationId mutationId, InetAddressAndPort fromHost)
     {
         Preconditions.checkArgument(!mutationId.isNone());
-        getOrCreate(keyspace).receivedWriteResponse(token, mutationId, onHost);
+        Shard shard = getShardNullable(mutationId);
+        // A response to the coordinator (for a forwarded write) won't have the coordinator log matching it
+        if (shard != null)
+            shard.receivedWriteResponse(mutationId, fromHost);
+//        outgoingMutations.receivedWriteResponse(mutationId, ClusterMetadata.current().directory.peerId(fromHost).id());
+    }
+
+    public void retryFailedWrite(ShortMutationId mutationId, InetAddressAndPort onHost, RequestFailure reason)
+    {
+        Preconditions.checkArgument(!mutationId.isNone());
+//        outgoingMutations.writeFailed(mutationId, reason, onHost);
+        activeReconciler.schedule(mutationId, onHost, ActiveLogReconciler.Priority.REGULAR);
     }
 
     public void updateReplicatedOffsets(String keyspace, Range<Token> range, List<? extends Offsets> offsets, InetAddressAndPort onHost)
     {
-        getOrCreate(keyspace).updateReplicatedOffsets(range, offsets, onHost);
+        getOrCreateShards(keyspace).updateReplicatedOffsets(range, offsets, onHost);
     }
 
-    public void startWriting(Mutation mutation)
+    public boolean startWriting(Mutation mutation)
     {
         Preconditions.checkArgument(!mutation.id().isNone());
-        getOrCreate(mutation.getKeyspaceName()).startWriting(mutation);
+        return getOrCreateShards(mutation.getKeyspaceName()).startWriting(mutation);
     }
 
     public void finishWriting(Mutation mutation)
     {
         Preconditions.checkArgument(!mutation.id().isNone());
-        getOrCreate(mutation.getKeyspaceName()).finishWriting(mutation);
+        getOrCreateShards(mutation.getKeyspaceName()).finishWriting(mutation);
+        incomingMutations.invokeListeners(mutation.id());
+    }
+
+    /**
+     * Register to be notified to an incoming mutation.
+     * @return true if this is the first active listener added for this id
+     */
+    public boolean registerMutationCallback(ShortMutationId mutationId, IncomingMutations.Callback callback)
+    {
+        return incomingMutations.subscribe(mutationId, callback);
     }
 
     public MutationSummary createSummaryForKey(DecoratedKey key, TableId tableId, boolean includePending)
     {
-        return getOrCreate(tableId).createSummaryForKey(key, tableId, includePending);
+        return getOrCreateShards(tableId).createSummaryForKey(key, tableId, includePending);
     }
 
     public MutationSummary createSummaryForRange(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending)
     {
-        return getOrCreate(tableId).createSummaryForRange(range, tableId, includePending);
+        return getOrCreateShards(tableId).createSummaryForRange(range, tableId, includePending);
     }
 
     public MutationSummary createSummaryForRange(Range<Token> range, TableId tableId, boolean includePending)
@@ -152,25 +209,62 @@ public class MutationTrackingService
 
     void forEachKeyspace(Consumer<KeyspaceShards> consumer)
     {
-        for (KeyspaceShards keyspaceShards : shards.values())
+        for (KeyspaceShards keyspaceShards : keyspaceShards.values())
             consumer.accept(keyspaceShards);
     }
 
-    private KeyspaceShards getOrCreate(TableId tableId)
+    public void collectLocallyMissingMutations(MutationSummary remoteSummary, Log2OffsetsMap.Mutable into)
+    {
+        Iterator<Offsets> iterator = remoteSummary.onlyUnreconciled();
+        while (iterator.hasNext())
+        {
+            Offsets offsets = iterator.next();
+            Shard shard = getShardNullable(offsets.logId);
+            if (shard == null)
+                into.add(offsets); // if the log/shard are unknown, then all the offsets are also unkown/missing
+            else
+                shard.collectLocallyMissingMutations(offsets, into);
+        }
+    }
+
+    public void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+    {
+        Shard shard = getShard(localOffsets.logId());
+        shard.collectRemotelyMissingMutations(localOffsets, remoteNodeIds, into);
+    }
+
+    public void requestMissingMutations(Offsets offsets, InetAddressAndPort forHost)
+    {
+        activeReconciler.schedule(offsets, forHost, ActiveLogReconciler.Priority.HIGH);
+    }
+
+    @Nullable
+    private Shard getShardNullable(CoordinatorLogId logId)
+    {
+        return log2ShardMap.get(logId);
+    }
+
+    @Nonnull
+    private Shard getShard(CoordinatorLogId logId)
+    {
+        return Preconditions.checkNotNull(log2ShardMap.get(logId));
+    }
+
+    private KeyspaceShards getOrCreateShards(TableId tableId)
     {
         //noinspection DataFlowIssue
-        return getOrCreate(Schema.instance.getTableMetadata(tableId).keyspace);
+        return getOrCreateShards(Schema.instance.getTableMetadata(tableId).keyspace);
     }
 
-    private KeyspaceShards getOrCreate(String keyspace)
+    private KeyspaceShards getOrCreateShards(String keyspace)
     {
-        KeyspaceShards ks = shards.get(keyspace);
+        KeyspaceShards ks = keyspaceShards.get(keyspace);
         if (ks != null)
             return ks;
 
         ClusterMetadata csm = ClusterMetadata.current();
         KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace);
-        return shards.computeIfAbsent(keyspace, ignore -> KeyspaceShards.make(ksm, csm, this::nextHostLogId));
+        return keyspaceShards.computeIfAbsent(keyspace, ignore -> KeyspaceShards.make(ksm, csm, this::nextHostLogId, this::onNewLog));
     }
 
     // TODO (expected): durability
@@ -185,8 +279,8 @@ public class MutationTrackingService
         // Could pass through SSTable bounds to exclude shards for non-overlapping ranges, but this will mostly be
         // called on flush for L0 SSTables with wide bounds.
 
-        KeyspaceShards keyspaceShards = shards.get(keyspace);
-        if (keyspaceShards == null)
+        KeyspaceShards shards = keyspaceShards.get(keyspace);
+        if (shards == null)
         {
             logger.debug("Could not find shards for keyspace {}", keyspace);
             return false;
@@ -195,7 +289,7 @@ public class MutationTrackingService
         for (Long logId : logOffsets)
         {
             CoordinatorLogId coordinatorLogId = new CoordinatorLogId(logId);
-            CoordinatorLog log = keyspaceShards.logs.get(coordinatorLogId);
+            CoordinatorLog log = shards.logs.get(coordinatorLogId);
             if (log == null)
             {
                 logger.warn("Could not determine lifecycle for unknown logId {}, not marking as durably reconciled", coordinatorLogId);
@@ -208,41 +302,66 @@ public class MutationTrackingService
         return true;
     }
 
+    // TODO (expected): when topology and state truncation is implemented, implement cleanup of this map as well
+    private void onNewLog(Shard shard, CoordinatorLog log)
+    {
+        log2ShardMap.put(log.logId, shard);
+    }
+
     private static class KeyspaceShards implements Shard.Subscriber
     {
         private final String keyspace;
         private final Map<Range<Token>, Shard> shards;
+        private final ReplicaGroups groups;
+        private final BiConsumer<Shard, CoordinatorLog> onNewLog;
 
         private transient final Map<Range<PartitionPosition>, Shard> ppShards;
         private transient final Map<CoordinatorLogId, CoordinatorLog> logs;
 
-        static KeyspaceShards make(KeyspaceMetadata keyspace, ClusterMetadata cluster, IntSupplier logIdProvider)
+        static KeyspaceShards make(KeyspaceMetadata keyspace, ClusterMetadata cluster, IntSupplier logIdProvider, BiConsumer<Shard, CoordinatorLog> onNewLog)
         {
             Preconditions.checkArgument(keyspace.params.replicationType.isTracked());
+
             Map<Range<Token>, Shard> shards = new HashMap<>();
-            cluster.placements.get(keyspace.params.replication).writes.forEach((tokenRange, forRange) -> {
+            Map<Range<Token>, VersionedEndpoints.ForRange> groups = new HashMap<>();
+
+            cluster.placements.get(keyspace.params.replication).writes.forEach((fullTokenRange, forRange) -> {
                 if (!forRange.endpoints().contains(FBUtilities.getBroadcastAddressAndPort()))
                     return;
-                IntArrayList participants = new IntArrayList(forRange.size(), IntArrayList.DEFAULT_NULL_VALUE);
+
+                IntArrayList participantList = new IntArrayList(forRange.size(), IntArrayList.DEFAULT_NULL_VALUE);
                 for (InetAddressAndPort endpoint : forRange.endpoints())
-                    participants.add(cluster.directory.peerId(endpoint).id());
-                Shard shard = new Shard(keyspace.name, tokenRange, cluster.myNodeId().id(), new Participants(participants), forRange.lastModified(), logIdProvider);
-                shards.put(tokenRange, shard);
+                    participantList.add(cluster.directory.peerId(endpoint).id());
+                Participants participants = new Participants(participantList);
+
+                Optional<Splitter> splitter = fullTokenRange.left.getPartitioner().splitter();
+                Set<Range<Token>> ranges = splitter.isPresent() && SHARD_MULTIPLIER > 1
+                                         ? splitter.get().split(fullTokenRange, SHARD_MULTIPLIER)
+                                         : Collections.singleton(fullTokenRange);
+
+                for (Range<Token> tokenRange : ranges)
+                {
+                    shards.put(tokenRange, new Shard(keyspace.name, tokenRange, cluster.myNodeId().id(), participants, forRange.lastModified(), logIdProvider, onNewLog));
+                    groups.put(tokenRange, forRange.map(original -> original.withRange(tokenRange)));
+                }
             });
-            return new KeyspaceShards(keyspace.name, shards);
+            return new KeyspaceShards(keyspace.name, shards, new ReplicaGroups(groups), onNewLog);
         }
 
-        KeyspaceShards(String keyspace, Map<Range<Token>, Shard> shards)
+        KeyspaceShards(String keyspace, Map<Range<Token>, Shard> shards, ReplicaGroups groups, BiConsumer<Shard, CoordinatorLog> onNewLog)
         {
             this.keyspace = keyspace;
             this.shards = shards;
+            this.groups = groups;
+            this.onNewLog = onNewLog;
 
-            this.ppShards = new HashMap<>();
             this.logs = new HashMap<>();
+            HashMap<Range<PartitionPosition>, Shard> ppShards = new HashMap<>();
             shards.forEach((range, shard) -> {
                 ppShards.put(Range.makeRowRange(range), shard);
                 shard.addSubscriber(this);
             });
+            this.ppShards = ppShards;
         }
 
         MutationId nextMutationId(Token token)
@@ -250,19 +369,14 @@ public class MutationTrackingService
             return lookUp(token).nextId();
         }
 
-        void receivedWriteResponse(Token token, MutationId mutationId, InetAddressAndPort onHost)
-        {
-            lookUp(token).receivedWriteResponse(mutationId, onHost);
-        }
-
         void updateReplicatedOffsets(Range<Token> range, List<? extends Offsets> offsets, InetAddressAndPort onHost)
         {
             shards.get(range).updateReplicatedOffsets(offsets, onHost);
         }
 
-        void startWriting(Mutation mutation)
+        boolean startWriting(Mutation mutation)
         {
-            lookUp(mutation).startWriting(mutation);
+            return lookUp(mutation).startWriting(mutation);
         }
 
         void finishWriting(Mutation mutation)
@@ -313,10 +427,7 @@ public class MutationTrackingService
 
         Shard lookUp(Token token)
         {
-            ClusterMetadata csm = ClusterMetadata.current();
-            KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace);
-            Range<Token> range = ClusterMetadata.current().placements.get(ksm.params.replication).writes.forRange(token).range();
-            return shards.get(range);
+            return shards.get(groups.forRange(token).range());
         }
 
         @Override
@@ -372,6 +483,15 @@ public class MutationTrackingService
             return executor.awaitTermination(timeout, units);
         }
 
+        public void shutdownBlocking() throws InterruptedException
+        {
+            if (!executor.isTerminated())
+            {
+                executor.shutdown();
+                executor.awaitTermination(1, MINUTES);
+            }
+        }
+
         @Override
         public void run()
         {
@@ -385,11 +505,11 @@ public class MutationTrackingService
 
         private void run(Shard shard)
         {
-            ShardReplicatedOffsets replicatedOffsets = shard.collectReplicatedOffsets();
+            BroadcastLogOffsets replicatedOffsets = shard.collectReplicatedOffsets();
             if (replicatedOffsets.isEmpty())
                 return;
 
-            Message<ShardReplicatedOffsets> message = Message.out(Verb.BROADCAST_LOG_OFFSETS, replicatedOffsets);
+            Message<BroadcastLogOffsets> message = Message.out(Verb.BROADCAST_LOG_OFFSETS, replicatedOffsets);
 
             for (InetAddressAndPort target : shard.remoteReplicas())
                 if (FailureDetector.instance.isAlive(target))
@@ -400,6 +520,18 @@ public class MutationTrackingService
     @VisibleForTesting
     public void broadcastOffsetsForTesting()
     {
-        broadcaster.run();
+        offsetsBroadcaster.run();
+    }
+
+    @VisibleForTesting
+    public void pauseActiveReconciler()
+    {
+        activeReconciler.pauseForTesting();
+    }
+
+    @VisibleForTesting
+    public void resumeActiveReconciler()
+    {
+        activeReconciler.resumeForTesting();
     }
 }
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
similarity index 52%
copy from src/java/org/apache/cassandra/replication/UnreconciledMutations.java
copy to src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
index a84bdc5162..12fa18c98a 100644
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
+++ b/src/java/org/apache/cassandra/replication/Node2OffsetsMap.java
@@ -17,25 +17,32 @@
  */
 package org.apache.cassandra.replication;
 
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.schema.TableId;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntObjConsumer;
 
 /**
- * Tracks unreconciled local mutations - the subset of all unreconciled mutations
- * that have been witnessed, or are currently being written to, on the local node.
+ * A mapping of nodes to Offsets; currently we only need an internally mutable implementation.
  */
-interface UnreconciledMutations
+public class Node2OffsetsMap
 {
-    void startWriting(Mutation mutation);
+    private final Int2ObjectHashMap<Offsets> map = new Int2ObjectHashMap<>();
 
-    void finishWriting(Mutation mutation);
+    public void add(int node, Offsets offsets)
+    {
+        Offsets.Mutable current = (Offsets.Mutable) map.get(node);
+        if (current != null)
+            current.addAll(offsets);
+        else
+            map.put(node, Offsets.Mutable.copy(offsets));
+    }
 
-    void remove(int offset);
+    public void forEach(IntObjConsumer<Offsets> consumer)
+    {
+        map.forEachInt(consumer);
+    }
 
-    boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
-
-    boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+    public void clear()
+    {
+        map.clear();
+    }
 }
diff --git a/src/java/org/apache/cassandra/replication/Offsets.java b/src/java/org/apache/cassandra/replication/Offsets.java
index 28e2d978f7..bd7204fc8a 100644
--- a/src/java/org/apache/cassandra/replication/Offsets.java
+++ b/src/java/org/apache/cassandra/replication/Offsets.java
@@ -27,6 +27,8 @@ import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.AbstractIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -35,6 +37,7 @@ import java.util.Iterator;
 
 public abstract class Offsets implements Iterable<ShortMutationId>
 {
+    private static final Logger logger = LoggerFactory.getLogger(Offsets.class);
     private static final int INITIAL_CAPACITY = 16;
 
     protected final CoordinatorLogId logId;
@@ -241,8 +244,26 @@ public abstract class Offsets implements Iterable<ShortMutationId>
 
         public void addAll(Offsets other, RangeConsumer onAdded)
         {
-            for (int i = 0; i < other.size; i += 2)
-                add(other.bounds[i], other.bounds[i + 1], onAdded);
+            if (size == 0)
+            {
+                int minLength = Math.max(other.size, INITIAL_CAPACITY);
+                if (bounds.length < minLength)
+                    bounds = new int[minLength];
+
+                System.arraycopy(other.bounds, 0, bounds, 0, other.size);
+                this.size = other.size;
+
+                if (onAdded != RangeConsumer.NONE)
+                {
+                    for (int i = 0; i < size; i += 2)
+                        onAdded.consume(logId, bounds[i], bounds[i + 1]);
+                }
+            }
+            else
+            {
+                for (int i = 0; i < other.size; i += 2)
+                    add(other.bounds[i], other.bounds[i + 1], onAdded);
+            }
         }
 
         public void addAll(Offsets other)
@@ -319,6 +340,44 @@ public abstract class Offsets implements Iterable<ShortMutationId>
             return true;
         }
 
+        /**
+         * Remove an offset in-place
+         */
+        public void remove(int offset)
+        {
+            if (size == 0)
+                return;
+
+            int pos = Arrays.binarySearch(bounds, 0, size, offset);
+            if (pos >= 0) // offset matches one of the bounds exactly
+            {
+                if (pos - 1 >= 0 && bounds[pos - 1] == offset)
+                    pos--; // normalize pos to always point to floor if this is a single-offset range
+
+                if (bounds[pos] == bounds[pos + 1]) // remove entire single-offset range
+                {
+                    System.arraycopy(bounds, pos + 2, bounds, pos, size - pos - 2);
+                    bounds[--size] = 0;
+                    bounds[--size] = 0;
+                }
+                else if (pos % 2 == 0)
+                {
+                    bounds[pos] = offset + 1; // offset is at a range start, shrink it
+                }
+                else
+                {
+                    bounds[pos] = offset - 1; // offset is at a range end, shrink it
+                }
+            }
+            else if ((-pos - 1) % 2 != 0) // offset falls within an existing range, need to split it into two
+            {
+                pos = -pos - 1;
+                insert(pos + 1, offset + 1, bounds[pos]);
+                bounds[pos] = offset - 1;
+            }
+            // the offset is before all existing ranges, in-between two, or after all exsiting ranges
+        }
+
         private enum AddAction
         {
             INSERT, MOVE, INCLUDE;
@@ -395,6 +454,17 @@ public abstract class Offsets implements Iterable<ShortMutationId>
 
             AddAction eMerge;
             {
+                // if the insert spans multiple ranges and the inserted end value
+                // is less than the start of eRange - 1 (ie: cant' be merged), decrement
+                // the range by one so we can extend the previous range
+                if (eRange > sRange) {
+                    int rStart = bounds[rangeStart(eRange)];
+
+                    if (end < rStart -1)
+                    {
+                        eRange--;
+                    }
+                }
                 int rStart = bounds[rangeStart(eRange)];
                 int rEnd = bounds[rangeEnd(eRange)];
 
diff --git a/src/java/org/apache/cassandra/replication/OutgoingMutations.java b/src/java/org/apache/cassandra/replication/OutgoingMutations.java
new file mode 100644
index 0000000000..901722800c
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/OutgoingMutations.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import org.agrona.collections.IntHashSet;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.jctools.maps.NonBlockingHashMapLong;
+
+/**
+ * Keep track of outgoing mutations so that we don't double-send any mutations
+ * when read reconciliation writes race with regular writes and/or each other.
+ * </p>
+ * Regular writes have a callback and need to be retried;
+ * Reconciliation writes have no callback, and only need to be retried if the writing node owns the log.
+ * </p>
+ * Note: this could live inside {@link CoordinatorLog}, since we cannot ever send a mutation
+ * that we don't have an instance of {@link CoordinatorLog} for, but for symmetry with
+ * {@link IncomingMutations} we put this inside {@link MutationTrackingService}.
+ */
+class OutgoingMutations
+{
+    private final NonBlockingHashMapLong<LogOutgoingMutations> log2MutationsMap = new NonBlockingHashMapLong<>();
+
+    void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+    {
+        getOrCreate(mutation.id()).sentWriteRequest(mutation, toHostIds);
+    }
+
+    void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
+    {
+        getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
+    }
+
+    void writeFailed(ShortMutationId mutationId, RequestFailureReason reason, InetAddressAndPort onHost)
+    {
+        getOrCreate(mutationId).writeFailed(mutationId, reason, onHost);
+    }
+
+    private LogOutgoingMutations getOrCreate(ShortMutationId mutationId)
+    {
+        LogOutgoingMutations mutations = log2MutationsMap.get(mutationId.logId());
+        if (mutations == null)
+            mutations = log2MutationsMap.computeIfAbsent(mutationId.logId(), ignore -> new LogOutgoingMutations());
+        return mutations;
+    }
+
+    private static class LogOutgoingMutations
+    {
+        // {host, offset} -> per-participant mutation state map
+        private final NonBlockingHashMapLong<OutgoingState> states = new NonBlockingHashMapLong<>();
+
+        void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
+        {
+            IntHashSet.IntIterator iterator = toHostIds.iterator();
+            while (iterator.hasNext())
+            {
+                OutgoingState state = getOrCreateState(iterator.nextValue(), mutation.id().offset());
+                state.setInFlight();
+            }
+        }
+
+        void receivedWriteResponse(ShortMutationId mutationId, int fromHostId)
+        {
+            OutgoingState state = remove(fromHostId, mutationId.offset());
+            if (state != null)
+                state.receivedWriteResponse();
+        }
+
+        void writeFailed(ShortMutationId mutationId, RequestFailureReason reason, InetAddressAndPort onHost)
+        {
+            // TODO check if need to implement this
+        }
+
+        // TODO FIXME: proper implementation
+        OutgoingState getOrCreateState(int hostId, int offset)
+        {
+            long key = key(hostId, offset);
+            OutgoingState state = states.get(key);
+            if (state != null)
+                return state;
+            return states.putIfAbsent(key, new OutgoingState());
+        }
+
+        OutgoingState remove(int hostId, int offset)
+        {
+            long key = key(hostId, offset);
+            return states.remove(key);
+        }
+
+        private long key(int hostId, int offset)
+        {
+            return ((long) hostId << 32) | offset;
+        }
+
+        private enum OutgoingStatus
+        {
+            IN_FLIGHT, ENQUEUED, DELIVERED
+        }
+
+        private static class OutgoingState
+        {
+            // TODO: callbacks for completion or failure
+            private volatile OutgoingStatus status;
+
+            void setInFlight()
+            {
+                status = OutgoingStatus.IN_FLIGHT;
+            }
+
+            void setEnqueued()
+            {
+                status = OutgoingStatus.ENQUEUED;
+            }
+
+            void receivedWriteResponse()
+            {
+                status = OutgoingStatus.DELIVERED;
+                // TODO: invoke all callbacks
+            }
+        }
+
+        private static class InFlight extends OutgoingState
+        {
+        }
+
+        private static class Queued extends OutgoingState
+        {
+        }
+
+        public interface Callback
+        {
+            void onSuccess();
+
+            void onFailure();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/replication/PullMutationsRequest.java b/src/java/org/apache/cassandra/replication/PullMutationsRequest.java
new file mode 100644
index 0000000000..ec5632ab5b
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/PullMutationsRequest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.IVersionedSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+
+public final class PullMutationsRequest
+{
+    private static final Logger logger = LoggerFactory.getLogger(PullMutationsRequest.class);
+
+    private final Offsets.Immutable offsets;
+
+    public PullMutationsRequest(Offsets.Immutable offsets)
+    {
+        this.offsets = offsets;
+    }
+
+    public static IVersionedSerializer<PullMutationsRequest> serializer = new IVersionedSerializer<>()
+    {
+        @Override
+        public void serialize(PullMutationsRequest pull, DataOutputPlus out, int version) throws IOException
+        {
+            Offsets.serializer.serialize(pull.offsets, out, version);
+        }
+
+        @Override
+        public PullMutationsRequest deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return new PullMutationsRequest(Offsets.serializer.deserialize(in, version));
+        }
+
+        @Override
+        public long serializedSize(PullMutationsRequest pull, int version)
+        {
+            return Offsets.serializer.serializedSize(pull.offsets, version);
+        }
+    };
+
+    public static IVerbHandler<PullMutationsRequest> verbHandler = new IVerbHandler<>()
+    {
+        @Override
+        public void doVerb(Message<PullMutationsRequest> message)
+        {
+            InetAddressAndPort forHost = message.from();
+            Offsets offsets = message.payload.offsets;
+            logger.trace("Received pull mutations request from {} for {}", forHost, offsets);
+            MutationTrackingService.instance.requestMissingMutations(offsets, forHost);
+        }
+    };
+}
diff --git a/src/java/org/apache/cassandra/replication/PushMutationRequest.java b/src/java/org/apache/cassandra/replication/PushMutationRequest.java
new file mode 100644
index 0000000000..8e673816b5
--- /dev/null
+++ b/src/java/org/apache/cassandra/replication/PushMutationRequest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.replication;
+
+import java.io.IOException;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.IVersionedAsymmetricSerializer;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.journal.RecordPointer;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.IVerbHandler;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.NoPayload;
+import org.apache.cassandra.tracing.Tracing;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.db.commitlog.CommitLogSegment.ENTRY_OVERHEAD_SIZE;
+import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
+
+public interface PushMutationRequest
+{
+    Logger logger = LoggerFactory.getLogger(PushMutationRequest.class);
+
+    long serializedSize(int version);
+    void serialize(DataOutputPlus out, int version) throws IOException;
+
+    class Referenced implements PushMutationRequest
+    {
+        private final ShortMutationId id;
+        private final RecordPointer pointer;
+
+        Referenced(ShortMutationId id, RecordPointer pointer)
+        {
+            this.id = id;
+            this.pointer = pointer;
+        }
+
+        @Override
+        public long serializedSize(int version)
+        {
+            // TODO (expected): handle mismatched (messaging) versions
+            int size = MutationJournal.instance.sizeOfRecord(pointer);
+            Preconditions.checkState(size > 0, "Couldn't read mutation %s size from the mutation journal", id);
+            return size;
+        }
+
+        @Override
+        public void serialize(DataOutputPlus out, int version) throws IOException
+        {
+            boolean read = MutationJournal.instance.read(pointer, (segment, position, key, buffer, userVersion) ->
+            {
+                try
+                {
+                    out.write(buffer); // TODO (expected): handle mismatched (messaging) versions
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            });
+
+            if (!read) throw new IllegalStateException("Couldn't find mutation " + id + " in the mutation journal");
+        }
+    }
+
+    class Materialized implements PushMutationRequest
+    {
+        public final Mutation mutation;
+
+        Materialized(Mutation mutation)
+        {
+            this.mutation = mutation;
+        }
+
+        @Override
+        public long serializedSize(int version)
+        {
+            return Mutation.serializer.serializedSize(mutation, version);
+        }
+
+        @Override
+        public void serialize(DataOutputPlus out, int version) throws IOException
+        {
+            Mutation.serializer.serialize(mutation, out, version);
+
+        }
+
+        static Materialized deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return new Materialized(Mutation.serializer.deserialize(in, version));
+        }
+    }
+
+    IVersionedAsymmetricSerializer<PushMutationRequest, Materialized> serializer = new IVersionedAsymmetricSerializer<>()
+    {
+        @Override
+        public long serializedSize(PushMutationRequest mutation, int version)
+        {
+            return mutation.serializedSize(version);
+        }
+
+        @Override
+        public void serialize(PushMutationRequest mutation, DataOutputPlus out, int version) throws IOException
+        {
+            mutation.serialize(out, version);
+        }
+
+        @Override
+        public Materialized deserialize(DataInputPlus in, int version) throws IOException
+        {
+            return Materialized.deserialize(in, version);
+        }
+    };
+
+    IVerbHandler<Materialized> verbHandler = new IVerbHandler<>()
+    {
+        @Override
+        public void doVerb(Message<Materialized> message)
+        {
+            if (approxTime.now() > message.expiresAtNanos())
+            {
+                Tracing.trace("Discarding mutation from {} (timed out)", message.from());
+                MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
+                return;
+            }
+
+            // TODO (expected): here and elsewhere, use Journal's size validation, not CommitLog's
+            message.payload.mutation.validateSize(MessagingService.current_version, ENTRY_OVERHEAD_SIZE);
+
+            try
+            {
+                applyMutation(message);
+            }
+            catch (WriteTimeoutException wto)
+            {
+                failed();
+            }
+        }
+
+        private void applyMutation(Message<Materialized> message)
+        {
+            Message<NoPayload> response = message.emptyResponse();
+            InetAddressAndPort respondTo = message.respondTo();
+            message.payload.mutation.applyFuture().addCallback(o -> respond(response, respondTo), wto -> failed());
+        }
+
+        private void respond(Message<NoPayload> response, InetAddressAndPort respondTo)
+        {
+            Tracing.trace("Enqueuing response to {}", respondTo);
+            logger.trace("Enqueuing response to {}", respondTo);
+            MessagingService.instance().send(response, respondTo);
+        }
+
+        private void failed()
+        {
+            Tracing.trace("Payload application resulted in WriteTimeout, not replying");
+        }
+    };
+}
diff --git a/src/java/org/apache/cassandra/replication/ReconciliationPlan.java b/src/java/org/apache/cassandra/replication/ReconciliationPlan.java
deleted file mode 100644
index ac085bf839..0000000000
--- a/src/java/org/apache/cassandra/replication/ReconciliationPlan.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.replication;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary;
-
-public class ReconciliationPlan
-{
-    private final ImmutableMap<InetAddressAndPort, Log2OffsetsMap.Immutable> txPlan;
-
-    public ReconciliationPlan(ImmutableMap<InetAddressAndPort, Log2OffsetsMap.Immutable> txPlan)
-    {
-        this.txPlan = txPlan;
-    }
-
-    public Set<InetAddressAndPort> nodes()
-    {
-        return txPlan.keySet();
-    }
-
-    public Log2OffsetsMap.Immutable peerReconciliation(InetAddressAndPort to)
-    {
-        return txPlan.get(to);
-    }
-
-    public Log2OffsetsMap.Immutable offsetsFor(InetAddressAndPort node)
-    {
-        return txPlan.get(node);
-    }
-
-    public boolean isEmpty()
-    {
-        return txPlan.isEmpty();
-    }
-
-    private static class PlanBuilder
-    {
-        final InetAddressAndPort node;
-
-        final MutationSummary summary;
-        final Map<InetAddressAndPort, Log2OffsetsMap.Immutable.Builder> peerReconciliations = new HashMap<>();
-
-        public PlanBuilder(InetAddressAndPort node, MutationSummary summary)
-        {
-            this.node = node;
-            this.summary = summary;
-        }
-
-        public void send(InetAddressAndPort to, Offsets sequenceIds)
-        {
-            peerReconciliations.computeIfAbsent(to, ep -> new Log2OffsetsMap.Immutable.Builder()).add(sequenceIds);
-        }
-
-        ReconciliationPlan build()
-        {
-            ImmutableMap.Builder<InetAddressAndPort, Log2OffsetsMap.Immutable> builder = ImmutableMap.builder();
-            peerReconciliations.forEach((to, ids) -> builder.put(to, ids.build()));
-            return new ReconciliationPlan(builder.build());
-        }
-    }
-
-    // TODO (desired): rework Offsets set logic usage to use Offsets#RangeIterator instead of rematerializing sets
-    private static class CoordinatorLogReconciliation
-    {
-        final CoordinatorLogId logId;
-        Offsets.Immutable reconciled;
-        Offsets.Immutable unreconciled;
-
-        Map<InetAddressAndPort, Offsets> unreconciledNodes = new HashMap<>();
-
-        CoordinatorLogReconciliation(CoordinatorLogId logId)
-        {
-            this.logId = logId;
-        }
-
-        void addPeerSummary(InetAddressAndPort peer, CoordinatorSummary summary)
-        {
-            Preconditions.checkArgument(summary.logId().equals(logId));
-            reconciled = Offsets.Immutable.union(reconciled, summary.reconciled);
-            unreconciled = Offsets.Immutable.union(unreconciled, summary.unreconciled);
-            unreconciledNodes.put(peer, summary.unreconciled);
-        }
-
-        void createPlan(Map<InetAddressAndPort, PlanBuilder> plan)
-        {
-            // remove reconciled ids
-            Offsets.Immutable allIds = Offsets.Immutable.difference(unreconciled, reconciled);
-            for (InetAddressAndPort receiver : plan.keySet())
-            {
-                Offsets.Immutable missing = Offsets.Immutable.difference(allIds, unreconciledNodes.get(receiver));
-                if (missing.isEmpty())
-                    continue;
-
-                // TODO: look into more intelligent ways to distribute mutation requests
-                for (Map.Entry<InetAddressAndPort, Offsets> sender : unreconciledNodes.entrySet())
-                {
-                    if (sender.getKey().equals(receiver))
-                        continue;
-
-                    Offsets senderIds = sender.getValue();
-                    PlanBuilder senderPlan = plan.get(sender.getKey());
-
-                    Offsets.Immutable requestedIds = Offsets.Immutable.intersection(missing, senderIds);
-                    senderPlan.send(receiver, requestedIds);
-
-                    missing = Offsets.Immutable.difference(missing, requestedIds);
-                    if (missing.rangeCount() == 0)
-                        break;
-                }
-            }
-        }
-    }
-
-    public static Map<InetAddressAndPort, ReconciliationPlan> calculateReconciliation(Map<InetAddressAndPort, MutationSummary> summaries)
-    {
-        Map<InetAddressAndPort, PlanBuilder> planBuilders = new HashMap<>();
-        Map<CoordinatorLogId, CoordinatorLogReconciliation> coordinatorReconciliations = new HashMap<>();
-
-        // organize data by peer and log id
-        summaries.forEach((node, summary) -> {
-
-            planBuilders.put(node, new PlanBuilder(node, summary));
-
-            for (int i=0; i<summary.size(); i++)
-            {
-                CoordinatorSummary coordinatorSummary = summary.get(i);
-                CoordinatorLogReconciliation reconciliation = coordinatorReconciliations.computeIfAbsent(coordinatorSummary.logId(), CoordinatorLogReconciliation::new);
-                reconciliation.addPeerSummary(node, coordinatorSummary);
-            }
-        });
-
-        coordinatorReconciliations.values().forEach(planBuilder -> planBuilder.createPlan(planBuilders));
-
-        Map<InetAddressAndPort, ReconciliationPlan> plans = new HashMap<>();
-        planBuilders.forEach((node, planBuilder) -> {
-            ReconciliationPlan plan = planBuilder.build();
-            if (!plan.isEmpty())
-                plans.put(node, plan);
-        });
-        return plans;
-    }
-
-    public static final IVersionedSerializer<ReconciliationPlan> serializer = new IVersionedSerializer<>()
-    {
-        @Override
-        public void serialize(ReconciliationPlan plan, DataOutputPlus out, int version) throws IOException
-        {
-            out.writeInt(plan.txPlan.size());
-            for (Map.Entry<InetAddressAndPort, Log2OffsetsMap.Immutable> entry : plan.txPlan.entrySet())
-            {
-                InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(entry.getKey(), out, version);
-                Log2OffsetsMap.Immutable.serializer.serialize(entry.getValue(), out, version);
-            }
-        }
-
-        @Override
-        public ReconciliationPlan deserialize(DataInputPlus in, int version) throws IOException
-        {
-            int size = in.readInt();
-            ImmutableMap.Builder<InetAddressAndPort, Log2OffsetsMap.Immutable> builder = ImmutableMap.builderWithExpectedSize(size);
-            for (int i = 0; i < size; i++)
-            {
-                InetAddressAndPort endpoint = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
-                Log2OffsetsMap.Immutable offsets = Log2OffsetsMap.Immutable.serializer.deserialize(in, version);
-                builder.put(endpoint, offsets);
-            }
-            return new ReconciliationPlan(builder.build());
-        }
-
-        @Override
-        public long serializedSize(ReconciliationPlan plan, int version)
-        {
-            long size = TypeSizes.sizeof(plan.txPlan.size());
-            for (Map.Entry<InetAddressAndPort, Log2OffsetsMap.Immutable> entry : plan.txPlan.entrySet())
-            {
-                size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(entry.getKey(), version);
-                size += Log2OffsetsMap.Immutable.serializer.serializedSize(entry.getValue(), version);
-            }
-            return size;
-        }
-    };
-}
diff --git a/src/java/org/apache/cassandra/replication/Shard.java b/src/java/org/apache/cassandra/replication/Shard.java
index 059ed84546..6588bede09 100644
--- a/src/java/org/apache/cassandra/replication/Shard.java
+++ b/src/java/org/apache/cassandra/replication/Shard.java
@@ -19,17 +19,20 @@ package org.apache.cassandra.replication;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.BiConsumer;
 import java.util.function.IntSupplier;
 
+import javax.annotation.Nonnull;
+
 import com.google.common.base.Preconditions;
 
+import org.agrona.collections.IntArrayList;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.replication.CoordinatorLog.CoordinatorLogPrimary;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.membership.NodeId;
@@ -42,11 +45,26 @@ public class Shard
     private final int localHostId;
     private final Participants participants;
     private final Epoch sinceEpoch;
+    private final BiConsumer<Shard, CoordinatorLog> onNewLog;
     private final NonBlockingHashMapLong<CoordinatorLog> logs;
     // TODO (expected): add support for log rotation
-    private final CoordinatorLogPrimary currentLocalLog;
+    private final CoordinatorLog.CoordinatorLogPrimary currentLocalLog;
+
+    private final List<Subscriber> subscribers = new ArrayList<>();
+
+    public interface Subscriber
+    {
+        default void onLogCreation(CoordinatorLog log) {}
+        default void onSubscribe(CoordinatorLog currentLog) {}
+    }
 
-    Shard(String keyspace, Range<Token> tokenRange, int localHostId, Participants participants, Epoch sinceEpoch, IntSupplier logIdProvider)
+    Shard(String keyspace,
+          Range<Token> tokenRange,
+          int localHostId,
+          Participants participants,
+          Epoch sinceEpoch,
+          IntSupplier logIdProvider,
+          BiConsumer<Shard, CoordinatorLog> onNewLog)
     {
         Preconditions.checkArgument(participants.contains(localHostId));
 
@@ -56,6 +74,7 @@ public class Shard
         this.participants = participants;
         this.sinceEpoch = sinceEpoch;
         this.logs = new NonBlockingHashMapLong<>();
+        this.onNewLog = onNewLog;
         this.currentLocalLog = startNewLog(localHostId, logIdProvider.getAsInt(), participants);
         CoordinatorLogId logId = currentLocalLog.logId;
         Preconditions.checkArgument(!logId.isNone());
@@ -67,10 +86,10 @@ public class Shard
         return currentLocalLog.nextId();
     }
 
-    void receivedWriteResponse(MutationId mutationId, InetAddressAndPort onHost)
+    void receivedWriteResponse(ShortMutationId mutationId, InetAddressAndPort fromHost)
     {
-        int onHostId = ClusterMetadata.current().directory.peerId(onHost).id();
-        getOrCreate(mutationId).receivedWriteResponse(mutationId, onHostId);
+        int fromHostId = ClusterMetadata.current().directory.peerId(fromHost).id();
+        getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
     }
 
     void updateReplicatedOffsets(List<? extends Offsets> offsets, InetAddressAndPort onHost)
@@ -80,14 +99,14 @@ public class Shard
             getOrCreate(logOffsets.logId()).updateReplicatedOffsets(logOffsets, onHostId);
     }
 
-    void startWriting(Mutation mutation)
+    boolean startWriting(Mutation mutation)
     {
-        getOrCreate(mutation.id()).startWriting(mutation);
+        return getOrCreate(mutation).startWriting(mutation);
     }
 
     void finishWriting(Mutation mutation)
     {
-        getOrCreate(mutation.id()).finishWriting(mutation);
+        getOrCreate(mutation).finishWriting(mutation);
     }
 
     void addSummaryForKey(Token token, boolean includePending, MutationSummary.Builder builder)
@@ -106,6 +125,18 @@ public class Shard
         });
     }
 
+    void collectLocallyMissingMutations(Offsets remoteOffsets, Log2OffsetsMap.Mutable into)
+    {
+        CoordinatorLog log = get(remoteOffsets.logId());
+        log.collectLocallyMissingMutations(remoteOffsets, into);
+    }
+
+    void collectRemotelyMissingMutations(Offsets localOffsets, IntArrayList remoteNodeIds, Node2OffsetsMap into)
+    {
+        CoordinatorLog log = get(localOffsets.logId());
+        log.collectRemotelyMissingMutations(localOffsets, remoteNodeIds, into);
+    }
+
     List<InetAddressAndPort> remoteReplicas()
     {
         List<InetAddressAndPort> replicas = new ArrayList<>(participants.size() - 1);
@@ -121,7 +152,7 @@ public class Shard
     /**
      * Collects replicated offsets for the logs owned by this coordinator on this shard.
      */
-    ShardReplicatedOffsets collectReplicatedOffsets()
+    BroadcastLogOffsets collectReplicatedOffsets()
     {
         List<Offsets.Immutable> offsets = new ArrayList<>();
         for (CoordinatorLog log : logs.values())
@@ -131,17 +162,25 @@ public class Shard
                 offsets.add(logOffsets);
         }
 
-        return new ShardReplicatedOffsets(keyspace, tokenRange, offsets);
+        return new BroadcastLogOffsets(keyspace, tokenRange, offsets);
     }
 
     /**
      * Creates a new coordinator log for this host. Primarily on Shard init (node startup or topology change).
      * Also on keyspace creation.
      */
-    private static CoordinatorLog.CoordinatorLogPrimary startNewLog(int localHostId, int hostLogId, Participants participants)
+    private CoordinatorLog.CoordinatorLogPrimary startNewLog(int localHostId, int hostLogId, Participants participants)
     {
         CoordinatorLogId logId = new CoordinatorLogId(localHostId, hostLogId);
-        return new CoordinatorLog.CoordinatorLogPrimary(localHostId, logId, participants);
+        CoordinatorLog.CoordinatorLogPrimary log =
+            new CoordinatorLog.CoordinatorLogPrimary(localHostId, logId, participants);
+        onNewLog.accept(this, log);
+        return log;
+    }
+
+    private CoordinatorLog getOrCreate(Mutation mutation)
+    {
+        return getOrCreate(mutation.id());
     }
 
     private CoordinatorLog getOrCreate(MutationId mutationId)
@@ -155,28 +194,36 @@ public class Shard
         return getOrCreate(logId.asLong());
     }
 
+    @Nonnull
+    private CoordinatorLog get(CoordinatorLogId logId)
+    {
+        return Preconditions.checkNotNull(logs.get(logId.asLong()));
+    }
+
     private CoordinatorLog getOrCreate(long logId)
     {
         CoordinatorLog log = logs.get(logId);
         if (log != null)
             return log;
         CoordinatorLog newLog = logs.computeIfAbsent(logId, ignore -> CoordinatorLog.create(localHostId, new CoordinatorLogId(logId), participants));
+        onNewLog.accept(this, newLog);
         for (Subscriber subscriber : subscribers)
             subscriber.onLogCreation(newLog);
         return newLog;
     }
 
-    private final List<Subscriber> subscribers = new ArrayList<>();
-
-    public interface Subscriber
-    {
-        default void onLogCreation(CoordinatorLog log) {}
-        default void onSubscribe(CoordinatorLog currentLog) {}
-    }
-
     public void addSubscriber(Subscriber subscriber)
     {
         subscriber.onSubscribe(currentLocalLog);
         subscribers.add(subscriber);
     }
+
+    private CoordinatorLog create(long logId)
+    {
+        CoordinatorLog log = CoordinatorLog.create(localHostId, new CoordinatorLogId(logId), participants);
+        onNewLog.accept(this, log);
+        for (Subscriber subscriber : subscribers)
+            subscriber.onLogCreation(log);
+        return log;
+    }
 }
diff --git a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
index 2673e3151d..e9ee3ce7cf 100644
--- a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
+++ b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.agrona.collections.IntHashSet;
 import org.apache.cassandra.concurrent.DebuggableTask;
 import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -55,6 +56,7 @@ import org.apache.cassandra.net.RequestCallback;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.service.AbstractWriteResponseHandler;
 import org.apache.cassandra.service.TrackedWriteResponseHandler;
+import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.FBUtilities;
@@ -101,10 +103,8 @@ public class TrackedWriteRequest
         writeMetrics.localRequests.mark();
         MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token);
         mutation = mutation.withMutationId(id);
-        TrackedWriteResponseHandler handler = TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime),
-                                         keyspaceName,
-                                         mutation.key().getToken(),
-                                         id);
+        TrackedWriteResponseHandler handler =
+            TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime), id);
         applyLocallyAndSendToReplicas(mutation, plan, handler);
         return handler;
     }
@@ -149,10 +149,10 @@ public class TrackedWriteRequest
 
             if (message == null)
             {
-                Message.Builder<Mutation> builder = Message.builder(MUTATION_REQ, mutation)
+                message = Message.builder(MUTATION_REQ, mutation)
                                  .withRequestTime(handler.getRequestTime())
-                                 .withFlag(MessageFlag.CALL_BACK_ON_FAILURE);
-                message = builder.build();
+                                 .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+                                 .build();
             }
 
             String dc = DatabaseDescriptor.getLocator().location(destination.endpoint()).datacenter;
@@ -178,16 +178,32 @@ public class TrackedWriteRequest
         Preconditions.checkState(applyLocally); // the coordinator is always a replica
         applyMutationLocally(mutation, handler);
 
+        IntHashSet remoteReplicas = null;
+        if (localDCReplicas != null || remoteDCReplicas != null)
+            remoteReplicas = new IntHashSet();
+
         if (localDCReplicas != null)
+        {
             for (Replica replica : localDCReplicas)
+            {
                 MessagingService.instance().sendWriteWithCallback(message, replica, handler);
+                remoteReplicas.add(ClusterMetadata.current().directory.peerId(replica.endpoint()).id());
+            }
+        }
 
         if (remoteDCReplicas != null)
         {
             // for each datacenter, send the message to one node to relay the write to other replicas
             for (List<Replica> dcReplicas : remoteDCReplicas.values())
+            {
                 sendMessagesToRemoteDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcReplicas), handler, null);
+                for (Replica replica : dcReplicas)
+                    remoteReplicas.add(ClusterMetadata.current().directory.peerId(replica.endpoint()).id());
+            }
         }
+
+        if (remoteReplicas != null)
+            MutationTrackingService.instance.sentWriteRequest(mutation, remoteReplicas);
     }
 
     static void applyMutationLocally(Mutation mutation, RequestCallback<NoPayload> handler)
@@ -268,9 +284,9 @@ public class TrackedWriteRequest
      * Send the message to the first replica of targets, and have it forward the message to others in its DC
      */
     static void sendMessagesToRemoteDC(Message<? extends IMutation> message,
-                                        EndpointsForToken targets,
-                                        RequestCallback<NoPayload> handler,
-                                        ForwardedWrite.CoordinatorAckInfo ackTo)
+                                       EndpointsForToken targets,
+                                       RequestCallback<NoPayload> handler,
+                                       ForwardedWrite.CoordinatorAckInfo ackTo)
     {
         final Replica target;
 
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
index a84bdc5162..740882b739 100644
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
+++ b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java
@@ -17,6 +17,16 @@
  */
 package org.apache.cassandra.replication;
 
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+
+import org.agrona.collections.Int2ObjectHashMap;
 import org.apache.cassandra.db.Mutation;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -27,15 +37,153 @@ import org.apache.cassandra.schema.TableId;
  * Tracks unreconciled local mutations - the subset of all unreconciled mutations
  * that have been witnessed, or are currently being written to, on the local node.
  */
-interface UnreconciledMutations
+public class UnreconciledMutations
 {
-    void startWriting(Mutation mutation);
+    private final Int2ObjectHashMap<Entry> statesMap = new Int2ObjectHashMap<>();
+    private final SortedSet<Entry> statesSet = new TreeSet<>(Entry.comparator);
+
+    enum Visibility
+    {
+        PENDING, // written to the journal, but not yet to LSM
+        VISIBLE, // written to both the journal and LSM
+    }
+
+    private static final class Entry
+    {
+        private static final Comparator<Entry> comparator = (left, right) ->
+        {
+            int cmp = left.token.compareTo(right.token);
+            return (cmp != 0) ? cmp : Integer.compare(left.offset, right.offset);
+        };
+
+        final Token token;
+        final int offset;
+        final Object tableOrTables;
+        private Visibility visibility;
+
+        Entry(Token token, int offset, Object tableOrTables, Visibility visibility)
+        {
+            this.token = token;
+            this.offset = offset;
+            this.tableOrTables = tableOrTables;
+            this.visibility = visibility;
+        }
+
+        static Entry create(Mutation mutation)
+        {
+            Collection<TableId> ids = mutation.getTableIds();
+            Preconditions.checkArgument(!ids.isEmpty());
+            return new Entry(mutation.key().getToken(), mutation.id().offset(), tableOrTables(mutation), Visibility.PENDING);
+        }
+
+        private static Object tableOrTables(Mutation mutation)
+        {
+            Collection<TableId> ids = mutation.getTableIds();
+            Preconditions.checkArgument(!ids.isEmpty());
+            return ids.size() == 1 ? ids.iterator().next() : Sets.newHashSet(mutation.getTableIds());
+        }
+
+        boolean contains(TableId tableId)
+        {
+            return tableOrTables instanceof Set
+                 ? ((Set<?>) tableOrTables).contains(tableId)
+                 : tableId.equals(tableOrTables);
+        }
+
+        boolean isVisible()
+        {
+            return visibility == Visibility.VISIBLE;
+        }
+
+        static Entry start(Token token, boolean isInclusive)
+        {
+            return new Entry(token, isInclusive ? 0 : Integer.MAX_VALUE, null, null);
+        }
+
+        static Entry end(Token token, boolean isInclusive)
+        {
+            return new Entry(token, isInclusive ? Integer.MAX_VALUE : 0, null, null);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (!(o instanceof Entry))
+                return false;
+            Entry that = (Entry) o;
+            return this.offset == that.offset && this.token.equals(that.token);
+        }
+    }
+
+    public void startWriting(Mutation mutation)
+    {
+        Entry entry = Entry.create(mutation);
+        statesMap.put(entry.offset, entry);
+        statesSet.add(entry);
+    }
+
+    public void finishWriting(Mutation mutation)
+    {
+        Preconditions.checkArgument(!mutation.id().isNone());
+        Entry entry = statesMap.get(mutation.id().offset());
+        if (entry == null)
+            return;
+        entry.visibility = Visibility.VISIBLE;
+    }
+
+    public void remove(int offset)
+    {
+        Entry state = statesMap.remove(offset);
+        Preconditions.checkNotNull(state);
+        statesSet.remove(state);
+    }
 
-    void finishWriting(Mutation mutation);
+    public boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+    {
+        Entry start = Entry.start(range.left.getToken(), range.left.kind() != PartitionPosition.Kind.MAX_BOUND);
+        Entry end = Entry.end(range.right.getToken(), range.right.kind() != PartitionPosition.Kind.MIN_BOUND);
+        return collect(start, end, tableId, includePending, into);
+    }
 
-    void remove(int offset);
+    public boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+    {
+        SortedSet<Entry> subset = statesSet.subSet(Entry.start(token, true), Entry.end(token, true));
+        return collect(subset, tableId, includePending, into);
+    }
 
-    boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+    private boolean collect(SortedSet<Entry> subset, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+    {
+        boolean found = false;
+        for (Entry entry : subset)
+        {
+            if (entry.contains(tableId) && (includePending || entry.isVisible()))
+            {
+                into.add(entry.offset);
+                found = true;
+            }
+        }
+        return found;
+    }
 
-    boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into);
+    private boolean collect(Entry start, Entry end, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
+    {
+        int cmp = start.token.compareTo(end.token);
+        if (cmp == 0)
+        {
+            // full range
+            return collect(statesSet, tableId, includePending, into);
+        }
+        else if (cmp > 0)
+        {
+            // wrap around range
+            boolean lFound = collect(statesSet.headSet(end), tableId, includePending, into);
+            boolean rFound = collect(statesSet.tailSet(start), tableId, includePending, into);
+            return lFound || rFound;
+        }
+        else
+        {
+            // contiguous range
+            return collect(statesSet.subSet(start, end), tableId, includePending, into);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java b/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java
deleted file mode 100644
index 821c8db0fc..0000000000
--- a/src/java/org/apache/cassandra/replication/UnreconciledMutationsReplica.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.replication;
-
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-
-import org.agrona.collections.Int2ObjectHashMap;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.PartitionPosition;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.schema.TableId;
-
-/**
- * Tracks unreconciled local mutations - the subset of all unreconciled mutations
- * that have been witnessed, or are currently being written to, on the local node.
- */
-class UnreconciledMutationsReplica implements UnreconciledMutations
-{
-    private final Int2ObjectHashMap<Entry> statesMap = new Int2ObjectHashMap<>();
-    private final SortedSet<Entry> statesSet = new TreeSet<>(Entry.comparator);
-
-    enum Visibility
-    {
-        PENDING, // written to the journal, but not yet to LSM
-        VISIBLE, // written to both the journal and LSM
-    }
-
-    private static final class Entry
-    {
-        private static final Comparator<Entry> comparator = (left, right) ->
-        {
-            int cmp = left.token.compareTo(right.token);
-            return (cmp != 0) ? cmp : Integer.compare(left.offset, right.offset);
-        };
-
-        final Token token;
-        final int offset;
-        final Object tableOrTables;
-        private Visibility visibility;
-
-        Entry(Token token, int offset, Object tableOrTables, Visibility visibility)
-        {
-            this.token = token;
-            this.offset = offset;
-            this.tableOrTables = tableOrTables;
-            this.visibility = visibility;
-        }
-
-        static Entry create(Mutation mutation)
-        {
-            Collection<TableId> ids = mutation.getTableIds();
-            Preconditions.checkArgument(!ids.isEmpty());
-            return new Entry(mutation.key().getToken(), mutation.id().offset(), tableOrTables(mutation), Visibility.PENDING);
-        }
-
-        private static Object tableOrTables(Mutation mutation)
-        {
-            Collection<TableId> ids = mutation.getTableIds();
-            Preconditions.checkArgument(!ids.isEmpty());
-            return ids.size() == 1 ? ids.iterator().next() : Sets.newHashSet(mutation.getTableIds());
-        }
-
-        boolean contains(TableId tableId)
-        {
-            return tableOrTables instanceof Set
-                 ? ((Set<?>) tableOrTables).contains(tableId)
-                 : tableId.equals(tableOrTables);
-        }
-
-        boolean isVisible()
-        {
-            return visibility == Visibility.VISIBLE;
-        }
-
-        static Entry start(Token token, boolean isInclusive)
-        {
-            return new Entry(token, isInclusive ? 0 : Integer.MAX_VALUE, null, null);
-        }
-
-        static Entry end(Token token, boolean isInclusive)
-        {
-            return new Entry(token, isInclusive ? Integer.MAX_VALUE : 0, null, null);
-        }
-
-        @Override
-        public boolean equals(Object o)
-        {
-            if (!(o instanceof Entry))
-                return false;
-            Entry that = (Entry) o;
-            return this.offset == that.offset && this.token.equals(that.token);
-        }
-    }
-
-    @Override
-    public void startWriting(Mutation mutation)
-    {
-        Entry entry = Entry.create(mutation);
-        statesMap.put(entry.offset, entry);
-        statesSet.add(entry);
-    }
-
-    @Override
-    public void finishWriting(Mutation mutation)
-    {
-        Preconditions.checkArgument(!mutation.id().isNone());
-        Entry entry = statesMap.get(mutation.id().offset());
-        Preconditions.checkNotNull(entry);
-        entry.visibility = Visibility.VISIBLE;
-    }
-
-    @Override
-    public void remove(int offset)
-    {
-        Entry state = statesMap.remove(offset);
-        Preconditions.checkNotNull(state);
-        statesSet.remove(state);
-    }
-
-    @Override
-    public boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
-    {
-        SortedSet<Entry> subset = statesSet.subSet(Entry.start(token, true), Entry.end(token, true));
-        return collect(subset, tableId, includePending, into);
-    }
-
-    @Override
-    public boolean collect(AbstractBounds<PartitionPosition> range, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
-    {
-        Entry start = Entry.start(range.left.getToken(), range.left.kind() != PartitionPosition.Kind.MAX_BOUND);
-        Entry end = Entry.end(range.right.getToken(), range.right.kind() != PartitionPosition.Kind.MIN_BOUND);
-        return collect(start, end, tableId, includePending, into);
-    }
-
-    private boolean collect(SortedSet<Entry> subset, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
-    {
-        boolean found = false;
-        for (Entry entry : subset)
-        {
-            if (entry.contains(tableId) && (includePending || entry.isVisible()))
-            {
-                into.add(entry.offset);
-                found = true;
-            }
-        }
-        return found;
-    }
-
-    private boolean collect(Entry start, Entry end, TableId tableId, boolean includePending, Offsets.OffsetReciever into)
-    {
-        int cmp = start.token.compareTo(end.token);
-        if (cmp == 0)
-        {
-            // full range
-            return collect(statesSet, tableId, includePending, into);
-        }
-        else if (cmp > 0)
-        {
-            // wrap around range
-            boolean lFound = collect(statesSet.headSet(end), tableId, includePending, into);
-            boolean rFound = collect(statesSet.tailSet(start), tableId, includePending, into);
-            return lFound || rFound;
-        }
-        else
-        {
-            // contiguous range
-            return collect(statesSet.subSet(start, end), tableId, includePending, into);
-        }
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java
index 3d41b2f955..b1559d17ba 100644
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@ -2722,7 +2722,10 @@ public class StorageProxy implements StorageProxyMBean
         ClusterMetadata metadata = ClusterMetadata.current();
 
         for (int i=0; i<cmdCount; i++)
-            reads[i] = TrackedRead.Partition.create(metadata, commands.get(i), consistencyLevel);
+        {
+            SinglePartitionReadCommand command = commands.get(i);
+            reads[i] = TrackedRead.Partition.create(metadata, command, consistencyLevel, requestTime);
+        }
 
         for (TrackedRead.Partition read : reads)
             read.start(requestTime);
diff --git a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
index f066182ad1..213cc90976 100644
--- a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
+++ b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java
@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.service;
 
-import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.RequestFailure;
 import org.apache.cassandra.exceptions.WriteFailureException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
@@ -31,24 +30,18 @@ public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler<No
 {
     private final AbstractWriteResponseHandler<NoPayload> wrapped;
 
-    private final String keyspace;
-    private final Token token;
     private final MutationId mutationId;
 
-    private TrackedWriteResponseHandler(
-        AbstractWriteResponseHandler<NoPayload> wrapped, String keyspace, Token token, MutationId mutationId)
+    private TrackedWriteResponseHandler(AbstractWriteResponseHandler<NoPayload> wrapped, MutationId mutationId)
     {
         super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, wrapped.getRequestTime());
         this.wrapped = wrapped;
-        this.keyspace = keyspace;
-        this.token = token;
         this.mutationId = mutationId;
     }
 
-    public static TrackedWriteResponseHandler wrap(
-        AbstractWriteResponseHandler<NoPayload> handler, String keyspace, Token token, MutationId mutationId)
+    public static TrackedWriteResponseHandler wrap(AbstractWriteResponseHandler<NoPayload> handler, MutationId mutationId)
     {
-        return new TrackedWriteResponseHandler(handler, keyspace, token, mutationId);
+        return new TrackedWriteResponseHandler(handler, mutationId);
     }
 
     @Override
@@ -56,13 +49,14 @@ public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler<No
     {
         // Local mutations are witnessed from Keyspace.applyInternalTracked
         if (msg != null)
-            MutationTrackingService.instance.receivedWriteResponse(keyspace, token, mutationId, msg.from());
+            MutationTrackingService.instance.receivedWriteResponse(mutationId, msg.from());
         wrapped.onResponse(msg);
     }
 
     @Override
     public void onFailure(InetAddressAndPort from, RequestFailure failure)
     {
+        MutationTrackingService.instance.retryFailedWrite(mutationId, from, failure);
         wrapped.onFailure(from, failure);
     }
 
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
index bf9885e224..4bb96b0128 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -71,6 +70,7 @@ import org.apache.cassandra.utils.CloseableIterator;
 import static com.google.common.base.Preconditions.checkState;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 @VisibleForTesting
@@ -342,7 +342,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
                 ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next();
                 PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), i == 0);
 
-                TrackedRead.Range read = TrackedRead.Range.create(rangeCommand, replicaPlan);
+                TrackedRead.Range read = TrackedRead.Range.create(rangeCommand, replicaPlan, requestTime);
                 read.start(requestTime);
                 concurrentQueries.add(read.iterator());
 
@@ -493,7 +493,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
             long latency = nanoTime() - requestTime.startedAtNanos();
             rangeMetrics.addNano(latency);
             rangeMetrics.roundTrips.update(batchesRequested);
-            Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
+            Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, NANOSECONDS);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
index f5fa4dc25a..88f4b496e8 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ExtendingCompletedRead.java
@@ -18,6 +18,7 @@
 
 package org.apache.cassandra.service.reads.tracked;
 
+import org.apache.cassandra.transport.Dispatcher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -132,7 +133,7 @@ class ExtendingCompletedRead implements PartialTrackedRead.CompletedRead
     }
 
     @Override
-    public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+    public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     {
         if (!followUpRequired())
             return null;
@@ -150,13 +151,14 @@ class ExtendingCompletedRead implements PartialTrackedRead.CompletedRead
         Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort());
         logger.info("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort());
 
-        return makeFollowupRead(initialResponse, toQuery, consistencyLevel, expiresAtNanos);
+        return makeFollowupRead(initialResponse, toQuery, consistencyLevel, requestTime);
     }
 
-    protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+    protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     {
-        TrackedRead.Range followUpRead = PartialTrackedRangeRead.makeFollowUpRead(command, followUpBounds, toQuery, consistencyLevel, expiresAtNanos);
-        followUpRead.start(expiresAtNanos);
+        TrackedRead.Range followUpRead = PartialTrackedRangeRead.makeFollowUpRead(command, followUpBounds, toQuery, consistencyLevel, requestTime);
+        logger.trace("Short read detected, starting followup read {}", followUpRead);
+        followUpRead.start(requestTime);
         AsyncPromise<TrackedDataResponse> combinedRead = new AsyncPromise<>();
         followUpRead.future().addCallback((result, failure) -> {
             if (failure != null)
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java b/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
index 121ce1a581..fe87dbad0f 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/FilteredFollowupRead.java
@@ -37,6 +37,7 @@ import org.apache.cassandra.db.partitions.PartitionIterator;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.service.reads.tracked.PartialTrackedRangeRead.FollowUpReadInfo;
 import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -50,7 +51,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
     private final TrackedDataResponse initialResponse;
     private final int toQuery;
     private final ConsistencyLevel consistencyLevel;
-    private final long expiresAtNanos;
+    private final Dispatcher.RequestTime requestTime;
     private final SortedMap<DecoratedKey, FollowUpReadInfo> followUpReadInfo;
     private final PartitionRangeReadCommand command;
     private final AbstractBounds<PartitionPosition> followUpBounds;
@@ -59,7 +60,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
     public FilteredFollowupRead(TrackedDataResponse initialResponse,
                                 int toQuery,
                                 ConsistencyLevel consistencyLevel,
-                                long expiresAtNanos,
+                                Dispatcher.RequestTime requestTime,
                                 SortedMap<DecoratedKey, FollowUpReadInfo> followUpReadInfo,
                                 PartitionRangeReadCommand command,
                                 AbstractBounds<PartitionPosition> followUpBounds,
@@ -68,7 +69,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
         this.initialResponse = initialResponse;
         this.toQuery = toQuery;
         this.consistencyLevel = consistencyLevel;
-        this.expiresAtNanos = expiresAtNanos;
+        this.requestTime = requestTime;
         this.followUpReadInfo = followUpReadInfo;
         this.command = command;
         this.followUpBounds = followUpBounds;
@@ -96,8 +97,8 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
             FollowUpReadInfo info = followUpReadInfo.get(key);
             remaining -= info.potentialMatches;
             SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fromRangeRead(key, command, command.limits().forShortReadRetry(toQuery));
-            TrackedRead.Partition read = TrackedRead.Partition.create(metadata, cmd, consistencyLevel);
-            read.start(expiresAtNanos);
+            TrackedRead.Partition read = TrackedRead.Partition.create(metadata, cmd, consistencyLevel, requestTime);
+            read.start(requestTime);
             futures.add(read.future());
         }
 
@@ -107,8 +108,8 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
         if (remaining > 0)
         {
             partialRead = new AtomicReference<>();
-            TrackedRead.Range rangeRead = makeFollowUpRead(command, followUpBounds, remaining, consistencyLevel, expiresAtNanos);
-            rangeRead.startLocal(expiresAtNanos, partialRead::set);
+            TrackedRead.Range rangeRead = makeFollowUpRead(command, followUpBounds, remaining, consistencyLevel, requestTime);
+            rangeRead.startLocal(requestTime, partialRead::set);
             futures.add(rangeRead.future());
         }
         else
@@ -153,7 +154,7 @@ class FilteredFollowupRead extends AsyncPromise<TrackedDataResponse>
                         PartialTrackedRangeRead followUpRangeRead = (PartialTrackedRangeRead) partialRead.get();
                         nextBounds = followUpRangeRead.followUpBounds();
                     }
-                    FilteredFollowupRead followUp = new FilteredFollowupRead(response, toQuery(command, mergedResultCounter), consistencyLevel, expiresAtNanos, nextKeys, command, nextBounds, null);
+                    FilteredFollowupRead followUp = new FilteredFollowupRead(response, toQuery(command, mergedResultCounter), consistencyLevel, requestTime, nextKeys, command, nextBounds, null);
                     followUp.start();
                     followUp.addCallback((result, failure) -> {
                         if (failure != null)
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
index b49570ceb7..2f3b58dfdc 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java
@@ -25,6 +25,7 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import com.google.common.base.Preconditions;
+import org.apache.cassandra.transport.Dispatcher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -305,7 +306,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
         {
             if (wasAugmented)
                 return extendRead(iterator);
-            return CompletedRead.simple(iterator, command);
+            return CompletedRead.simple(iterator, command, command.nowInSec());
         }
 
         AbstractBounds<PartitionPosition> followUpBounds()
@@ -354,7 +355,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
         return completed.followUpBounds();
     }
 
-    protected static TrackedRead.Range makeFollowUpRead(PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> followUpBounds, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+    protected static TrackedRead.Range makeFollowUpRead(PartitionRangeReadCommand command, AbstractBounds<PartitionPosition> followUpBounds, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     {
         DataLimits newLimits = command.limits().forShortReadRetry(toQuery);
 
@@ -369,7 +370,7 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
                                                                          followUpCmd.dataRange().keyRange(),
                                                                          1);
 
-        TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan);
+        TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan, requestTime);
         logger.trace("Short read detected, starting followup read {}", read);
         return read;
     }
@@ -556,15 +557,15 @@ public abstract class PartialTrackedRangeRead extends AbstractPartialTrackedRead
             }
 
             @Override
-            protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+            protected Future<TrackedDataResponse> makeFollowupRead(TrackedDataResponse initialResponse, int toQuery, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
             {
                 if (followUpReadInfo.isEmpty())
-                    return super.makeFollowupRead(initialResponse, toQuery, consistencyLevel, expiresAtNanos);
+                    return super.makeFollowupRead(initialResponse, toQuery, consistencyLevel, requestTime);
 
                 FilteredFollowupRead followupRead = new FilteredFollowupRead(initialResponse,
                                                                              toQuery,
                                                                              consistencyLevel,
-                                                                             expiresAtNanos,
+                                                                             requestTime,
                                                                              followUpReadInfo,
                                                                              command,
                                                                              followUpBounds,
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
index 99e509ee7f..cd3b137b6f 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java
@@ -18,7 +18,7 @@
 
 package org.apache.cassandra.service.reads.tracked;
 
-import java.util.Collection;
+import com.google.common.base.Preconditions;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -31,13 +31,17 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.utils.concurrent.Future;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationJournal;
+import org.apache.cassandra.replication.ShortMutationId;
+import org.apache.cassandra.transport.Dispatcher;
 
 public interface PartialTrackedRead
 {
     interface CompletedRead extends AutoCloseable
     {
         TrackedDataResponse response(); // must be called from the read stage
-        Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, long expiresAtNanos);
+        Future<TrackedDataResponse> followupRead(TrackedDataResponse initialResponse, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime);
 
         @Override
         void close();
@@ -53,7 +57,7 @@ public interface PartialTrackedRead
                                               command.columnFilter());
         }
 
-        static CompletedRead simple(UnfilteredPartitionIterator partition, ReadCommand command)
+        static CompletedRead simple(UnfilteredPartitionIterator partition, ReadCommand command, long nowInSec)
         {
             return new CompletedRead()
             {
@@ -64,7 +68,7 @@ public interface PartialTrackedRead
                 }
 
                 @Override
-                public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialRead, ConsistencyLevel consistencyLevel, long expiresAtNanos)
+                public Future<TrackedDataResponse> followupRead(TrackedDataResponse initialRead, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
                 {
                     return null;
                 }
@@ -82,9 +86,16 @@ public interface PartialTrackedRead
 
     void augment(Mutation mutation);
 
-    default void augment(Collection<Mutation> mutations)
+    default void augment(Log2OffsetsMap<?> augmentingOffsets)
     {
-        mutations.forEach(this::augment);
+        augmentingOffsets.forEach(this::augment);
+    }
+
+    default void augment(ShortMutationId mutationId)
+    {
+        Mutation mutation = MutationJournal.instance.read(mutationId);
+        Preconditions.checkNotNull(mutation);
+        augment(mutation);
     }
 
     ReadExecutionController executionController();
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
index e15dba8a5c..368a4f0bfd 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java
@@ -126,7 +126,7 @@ public class PartialTrackedSinglePartitionRead extends AbstractPartialTrackedRea
         @Override
         protected CompletedRead createResult(UnfilteredPartitionIterator iterator)
         {
-            return CompletedRead.simple(iterator, command);
+            return CompletedRead.simple(iterator, command, command.nowInSec());
         }
     }
 
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
similarity index 50%
rename from src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java
rename to src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
index 6a72a7a01a..bfa716a9e0 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileAck.java
@@ -15,7 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.service.reads.tracked;
 
 import java.io.IOException;
@@ -23,73 +22,58 @@ import java.io.IOException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.replication.MutationTrackingService;
 
 /**
- * Notifies the read coordinator that this node has received mutations
- * from the sending node
+ * Notifies the reconcile coordinator (data node) that this node has received
+ * all missing mutations from involved peers that were needed for the read.
  */
-public class ReadReconcileNotify
+public class ReadReconcileAck
 {
-    private static final Logger logger = LoggerFactory.getLogger(ReadReconcileNotify.class);
+    private static final Logger logger = LoggerFactory.getLogger(ReadReconcileAck.class);
 
     public final TrackedRead.Id readId;
-    public final int syncId;
 
-    public ReadReconcileNotify(TrackedRead.Id readId, int syncId)
+    public ReadReconcileAck(TrackedRead.Id readId)
     {
         this.readId = readId;
-        this.syncId = syncId;
     }
 
     @Override
     public String toString()
     {
-        return "ReadReconcileNotify{" +
-               "reconciliationId=" + readId +
-               ", syncId=" + syncId +
-               '}';
+        return "ReadReconcileAck{readId=" + readId + '}';
     }
 
-    public static final IVerbHandler<ReadReconcileNotify> verbHandler = new IVerbHandler<ReadReconcileNotify>()
+    public static final IVerbHandler<ReadReconcileAck> verbHandler = message ->
     {
-        @Override
-        public void doVerb(Message<ReadReconcileNotify> message) throws IOException
-        {
-            ReadReconcileNotify notify = message.payload;
-            logger.trace("Received read reconcile notify from {}: {}", message.from(), notify);
-            MutationTrackingService.instance.localReads().acknowledgeSync(notify.readId, notify.syncId);
-        }
+        ReadReconcileAck notify = message.payload;
+        logger.trace("Received reconcile ack from {}, for {}", message.from(), notify.readId);
+        ReadReconciliations.instance.acceptSyncAck(message.from(), notify.readId);
     };
 
-    public static final IVersionedSerializer<ReadReconcileNotify> serializer = new IVersionedSerializer<>()
+    public static final IVersionedSerializer<ReadReconcileAck> serializer = new IVersionedSerializer<>()
     {
         @Override
-        public void serialize(ReadReconcileNotify notify, DataOutputPlus out, int version) throws IOException
+        public void serialize(ReadReconcileAck notify, DataOutputPlus out, int version) throws IOException
         {
             TrackedRead.Id.serializer.serialize(notify.readId, out, version);
-            out.writeInt(notify.syncId);
         }
 
         @Override
-        public ReadReconcileNotify deserialize(DataInputPlus in, int version) throws IOException
+        public ReadReconcileAck deserialize(DataInputPlus in, int version) throws IOException
         {
             TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
-            int syncId = in.readInt();
-            return new ReadReconcileNotify(readId, syncId);
+            return new ReadReconcileAck(readId);
         }
 
         @Override
-        public long serializedSize(ReadReconcileNotify notify, int version)
+        public long serializedSize(ReadReconcileAck notify, int version)
         {
-            return TrackedRead.Id.serializer.serializedSize(notify.readId, version)
-                   + TypeSizes.sizeof(notify.syncId);
+            return TrackedRead.Id.serializer.serializedSize(notify.readId, version);
         }
     };
 }
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java
deleted file mode 100644
index 6a5fd7db52..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.cassandra.service.reads.tracked;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.MutationTrackingService;
-import org.apache.cassandra.utils.CollectionSerializers;
-
-import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
-
-public class ReadReconcileReceive
-{
-    private static final Logger logger = LoggerFactory.getLogger(ReadReconcileReceive.class);
-
-    public final TrackedRead.Id readId;
-    public final int syncId;
-    public final InetAddressAndPort coordinator;
-    public final List<Mutation> mutations;
-
-    public ReadReconcileReceive(TrackedRead.Id readId, int syncId, InetAddressAndPort coordinator, List<Mutation> mutations)
-    {
-        this.readId = readId;
-        this.syncId = syncId;
-        this.coordinator = coordinator;
-        this.mutations = mutations;
-    }
-
-    private static String mutationString(List<Mutation> mutations)
-    {
-        StringBuilder builder = new StringBuilder('[');
-        boolean isFirst = true;
-        for (Mutation mutation : mutations)
-        {
-            if (!isFirst)
-            {
-                builder.append(", ");
-            }
-            isFirst = false;
-            builder.append(mutation.id());
-        }
-        builder.append(']');
-        return builder.toString();
-    }
-
-    @Override
-    public String toString()
-    {
-        return "ReadReconcileReceive{" +
-               "reconciliationId=" + readId +
-               ", syncId=" + syncId +
-               ", coordinator=" + coordinator +
-               ", mutations=" + mutationString(mutations) +
-               '}';
-    }
-
-    public static final IVerbHandler<ReadReconcileReceive> verbHandler = new IVerbHandler<ReadReconcileReceive>()
-    {
-        @Override
-        public void doVerb(Message<ReadReconcileReceive> message) throws IOException
-        {
-            // TODO: check epoch and tokens?
-            ReadReconcileReceive receive = message.payload;
-            logger.trace("Received read reconciliation from {}: {}", message.from(), receive);
-            receive.mutations.forEach(Mutation::apply);
-
-            if (!MutationTrackingService.instance.localReads().receiveMutations(receive.readId, receive.syncId, receive.mutations))
-            {
-                // if this isn't a locally coordinated read, notify the coordinator
-                ReadReconcileNotify notify = new ReadReconcileNotify(receive.readId, receive.syncId);
-                MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_NOTIFY, notify), receive.coordinator);
-            }
-        }
-    };
-
-    public static final IVersionedSerializer<ReadReconcileReceive> serializer = new IVersionedSerializer<ReadReconcileReceive>()
-    {
-        @Override
-        public void serialize(ReadReconcileReceive rcv, DataOutputPlus out, int version) throws IOException
-        {
-            TrackedRead.Id.serializer.serialize(rcv.readId, out, version);
-            out.writeInt(rcv.syncId);
-            inetAddressAndPortSerializer.serialize(rcv.coordinator, out, version);
-            CollectionSerializers.serializeCollection(rcv.mutations, out, version, Mutation.serializer);
-
-        }
-
-        @Override
-        public ReadReconcileReceive deserialize(DataInputPlus in, int version) throws IOException
-        {
-            TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
-            int syncId = in.readInt();
-            InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version);
-            List<Mutation> mutations = CollectionSerializers.deserializeList(in, version, Mutation.serializer);
-            return new ReadReconcileReceive(readId, syncId, coordinator, mutations);
-        }
-
-        @Override
-        public long serializedSize(ReadReconcileReceive t, int version)
-        {
-            return TrackedRead.Id.serializer.serializedSize(t.readId, version)
-                   + TypeSizes.sizeof(t.syncId)
-                   + inetAddressAndPortSerializer.serializedSize(t.coordinator, version)
-                   + CollectionSerializers.serializedCollectionSize(t.mutations, version, Mutation.serializer);
-        }
-    };
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java
deleted file mode 100644
index d89223f6cc..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.reads.tracked;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.TypeSizes;
-import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.net.IVerbHandler;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.Log2OffsetsMap;
-import org.apache.cassandra.replication.MutationJournal;
-import org.apache.cassandra.utils.CollectionSerializers;
-
-/**
- * Instructs a node to send mutations to other node
- */
-public class ReadReconcileSend
-{
-    private static final Logger logger = LoggerFactory.getLogger(ReadReconcileSend.class);
-
-    public static class PeerSync
-    {
-        final int syncId;
-        final InetAddressAndPort to;
-        final Log2OffsetsMap.Immutable plan;
-
-        public PeerSync(int syncId, InetAddressAndPort to, Log2OffsetsMap.Immutable plan)
-        {
-            this.syncId = syncId;
-            this.to = to;
-            this.plan = plan;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "PeerSync{" +
-                   "syncId=" + syncId +
-                   ", to=" + to +
-                   ", plan=" + plan +
-                   '}';
-        }
-
-        public static final IVersionedSerializer<PeerSync> serializer = new IVersionedSerializer<PeerSync>()
-        {
-            @Override
-            public void serialize(PeerSync sync, DataOutputPlus out, int version) throws IOException
-            {
-                out.writeInt(sync.syncId);
-                InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(sync.to, out, version);
-                Log2OffsetsMap.Immutable.serializer.serialize(sync.plan, out, version);
-            }
-
-            @Override
-            public PeerSync deserialize(DataInputPlus in, int version) throws IOException
-            {
-                return new PeerSync(in.readInt(),
-                                    InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version),
-                                    Log2OffsetsMap.Immutable.serializer.deserialize(in, version));
-            }
-
-            @Override
-            public long serializedSize(PeerSync sync, int version)
-            {
-                return TypeSizes.sizeof(sync.syncId)
-                       + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(sync.to, version)
-                       + Log2OffsetsMap.Immutable.serializer.serializedSize(sync.plan, version);
-            }
-        };
-    }
-
-    public final TrackedRead.Id reconcileId;
-    public final ImmutableList<PeerSync> syncTasks;
-
-    public ReadReconcileSend(TrackedRead.Id reconcileId, List<PeerSync> syncTasks)
-    {
-        this.reconcileId = reconcileId;
-        this.syncTasks = ImmutableList.copyOf(syncTasks);
-    }
-
-    @Override
-    public String toString()
-    {
-        return "ReadReconcileSend{" +
-               "reconcileId=" + reconcileId +
-               ", syncTasks=" + syncTasks +
-               '}';
-    }
-
-    public static final IVerbHandler<ReadReconcileSend> verbHandler = new IVerbHandler<>()
-    {
-        @Override
-        public void doVerb(Message<ReadReconcileSend> message)
-        {
-            logger.trace("Received ReadReconcileSend from {}: {}", message.from(), message.payload);
-            // TODO: check epoch and tokens?
-            ReadReconcileSend payload = message.payload;
-            for (PeerSync sync : message.payload.syncTasks)
-            {
-                // TODO (expected): do not deser just to serialize again, if same messaging versions (common case)
-                // TODO (expected): don't materialize mutation ids, look up from offset collections
-                int mutationCount = sync.plan.idCount();
-                List<Mutation> mutations = new ArrayList<>(mutationCount);
-                MutationJournal.instance.readAll(sync.plan, mutations);
-                Preconditions.checkArgument(mutationCount == mutations.size());
-
-                ReadReconcileReceive receive = new ReadReconcileReceive(payload.reconcileId, sync.syncId, message.from(), mutations);
-                logger.trace("Sending {} to replica {}", receive, sync.to);
-                MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_RCV, receive), sync.to);
-            }
-        }
-    };
-
-    public static final IVersionedSerializer<ReadReconcileSend> serializer = new IVersionedSerializer<>()
-    {
-        @Override
-        public void serialize(ReadReconcileSend send, DataOutputPlus out, int version) throws IOException
-        {
-            TrackedRead.Id.serializer.serialize(send.reconcileId, out, version);
-            CollectionSerializers.serializeList(send.syncTasks, out, version, PeerSync.serializer);
-        }
-
-        @Override
-        public ReadReconcileSend deserialize(DataInputPlus in, int version) throws IOException
-        {
-            TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version);
-            List<PeerSync> syncTasks = CollectionSerializers.deserializeList(in, version, PeerSync.serializer);
-            return new ReadReconcileSend(readId, syncTasks);
-        }
-
-        @Override
-        public long serializedSize(ReadReconcileSend send, int version)
-        {
-            return TrackedRead.Id.serializer.serializedSize(send.reconcileId, version) + CollectionSerializers.serializedCollectionSize(send.syncTasks, version, PeerSync.serializer);
-        }
-    };
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java
new file mode 100644
index 0000000000..17851a24e3
--- /dev/null
+++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconciliations.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service.reads.tracked;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.net.Message;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.net.Verb;
+import org.apache.cassandra.replication.CoordinatorLogId;
+import org.apache.cassandra.replication.ExpiredStatePurger;
+import org.apache.cassandra.replication.IncomingMutations;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationSummary;
+import org.apache.cassandra.replication.MutationTrackingService;
+import org.apache.cassandra.replication.Node2OffsetsMap;
+import org.apache.cassandra.replication.Offsets;
+import org.apache.cassandra.replication.PullMutationsRequest;
+import org.apache.cassandra.replication.ShortMutationId;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Accumulator;
+import org.jctools.maps.NonBlockingHashMap;
+import org.jctools.maps.NonBlockingHashSet;
+
+/**
+ * On the initial coordinator (replica or not):
+ *  0. Issue summary requests to all summary nodes, in parallel
+ * <p>
+ * On the data node / coordinator:
+ *  1. Compute the secondary summary, send to all other nodes for reconcile
+ *  2. For primary logs in the summary, initiate priority delivery of mutations
+ *     that are present in our summary but absent from any summary node
+ *  3. Collect summary responses from all summary nodes, request missing mutations
+ *     whenever we get a summary from any of the nodes
+ *  4. Collect the requested mutations from other nodes
+ *  5. Collect sync-done acks from every summary node
+ *  6. When all the summaries, all missing mutations, and all sync-done acks have
+ *     been received, invoke the reconcile-complete callback
+ *  7. Clean up state after ourselves (states map)
+ * <p>
+ * On each summary node:
+ *  1. Compute the secondary summary, send to all other nodes for reconcile
+ *  2. For primary logs in the summary, initiate priority delivery of mutations
+ *     that are present in our summary but absent from any summary or data node
+ *  3. Collect summary responses from all summary nodes, request missing mutations
+ *     whenever we get a summary from any of the nodes
+ *  4. Collect the requested mutations from other nodes
+ *  5. When all the summaries and all missing mutations have been received,
+ *     respond to the data node with a sync-done message
+ *  6. Clean up state after ourselves (states) map
+ */
+public class ReadReconciliations implements ExpiredStatePurger.Expireable
+{
+    private static final int LOCAL_NODE = ClusterMetadata.current().myNodeId().id();
+
+    public static final ReadReconciliations instance = new ReadReconciliations();
+
+    // We are reusing the read id just for the ease of it, but it doesn't really have to match
+    // read's id. Could be another instance of it, or a UUID or any other cluster-unique identifier.
+    private final NonBlockingHashMap<TrackedRead.Id, Coordinator> coordinators = new NonBlockingHashMap<>();
+
+    /**
+     * Main entry point for summary nodes - if a coordinator's request arrives before any peer responses.
+     * Executed in READ stage. Calculate the summary, process it locally and send it to peers so that they
+     * can fill their own gaps.
+     */
+    public void handleSummaryRequest(TrackedRead.SummaryRequest request)
+    {
+        Coordinator reconcile = getOrCreateCoordinator(request.readId, request.dataNode, request.summaryNodes);
+        if (reconcile.acceptLocalSummary(request.command.createMutationSummary(false)))
+            coordinators.remove(request.readId);
+    }
+
+    /**
+     * The way for the data node to provide its final (seconary) summary.
+     */
+    public void acceptLocalSummary(TrackedRead.Id id, MutationSummary summary, int[] summaryNodes)
+    {
+        Coordinator reconcile = getOrCreateCoordinator(id, LOCAL_NODE, summaryNodes);
+        if (reconcile.acceptLocalSummary(summary))
+            coordinators.remove(id);
+    }
+
+    /**
+     * Can arrive from a peer node before a summary request
+     */
+    public void acceptRemoteSummary(InetAddressAndPort from, TrackedSummaryResponse summary)
+    {
+        Coordinator reconcile = getOrCreateCoordinator(summary.readId, summary.dataNode, summary.summaryNodes);
+        if (reconcile.acceptRemoteSummary(summary.summary, node(from)))
+            coordinators.remove(summary.readId);
+    }
+
+    public void acceptSyncAck(InetAddressAndPort from, TrackedRead.Id id)
+    {
+        Coordinator reconcile = coordinators.get(id);
+        if (reconcile != null && reconcile.acceptSyncAck(node(from))) // could be already timed out / expired
+            coordinators.remove(id);
+    }
+
+    public void acceptMutation(TrackedRead.Id id, ShortMutationId mutationId)
+    {
+        Coordinator reconcile = coordinators.get(id);
+        if (reconcile != null && reconcile.acceptMutation(mutationId)) // could be already timed out / expired
+            coordinators.remove(id);
+    }
+
+    private Coordinator getOrCreateCoordinator(TrackedRead.Id id, int dataNode, int[] summaryNodes)
+    {
+        Coordinator reconcile = coordinators.get(id);
+        if (reconcile != null) return reconcile;
+        return coordinators.computeIfAbsent(id, readId -> new Coordinator(id, dataNode, summaryNodes));
+    }
+
+    @Override
+    public int expire(long nanoTime)
+    {
+        int n = 0;
+        for (Map.Entry<TrackedRead.Id, Coordinator> entry : coordinators.entrySet())
+        {
+            TrackedRead.Id id = entry.getKey();
+            Coordinator coordinator = entry.getValue();
+
+            if (coordinator.isPurgeable(nanoTime) && coordinators.remove(id, coordinator))
+                n++;
+        }
+        return n;
+    }
+
+    private static final class Coordinator
+    {
+        private static final AtomicLongFieldUpdater<Coordinator> remainingUpdater =
+            AtomicLongFieldUpdater.newUpdater(Coordinator.class, "remaining");
+        private volatile long remaining; // three values packed into one atomic long
+
+        private final TrackedRead.Id id;
+        private final int dataNode;
+        private final int[] summaryNodes;
+        private final long expiresAtNanos;
+
+        private final Accumulator<Pair<Integer, MutationSummary>> summaries;
+
+        private final Set<ShortMutationId> requested = new NonBlockingHashSet<>();
+
+        Coordinator(TrackedRead.Id id, int dataNode, int[] summaryNodes)
+        {
+            this.id = id;
+            this.dataNode = dataNode;
+            this.summaryNodes = summaryNodes;
+            this.expiresAtNanos =
+                Clock.Global.nanoTime() + Math.max(DatabaseDescriptor.getReadRpcTimeout(TimeUnit.NANOSECONDS),
+                                                   DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.NANOSECONDS));
+            int remainingSummaries = 1 + summaryNodes.length;
+            int remainingMutations = 0;
+            int remainingSyncAcks = dataNode == LOCAL_NODE ? summaryNodes.length : 0;
+            remaining = remaining(remainingMutations, remainingSummaries, remainingSyncAcks);
+
+            summaries = new Accumulator<>(remainingSummaries);
+        }
+
+        /**
+         * For all the logs in the summary that are owned by us, preemptively prioritise delivery of
+         * any mutations that are absent from other participating nodes according to our primary coordinator
+         * log knowledge. This is an optimisation, and it can be omitted without affecting correctness.
+         */
+        boolean acceptLocalSummary(MutationSummary summary)
+        {
+            IntArrayList remoteNodes = new IntArrayList(summaryNodes.length, Integer.MIN_VALUE);
+            if (dataNode != LOCAL_NODE)
+                remoteNodes.addInt(dataNode);
+            for (int node : summaryNodes)
+                if (node != LOCAL_NODE)
+                    remoteNodes.addInt(node);
+
+            if (!remoteNodes.isEmpty())
+            {
+                // send the summary to all peers, so that they can initiate reconciling their gaps ASAP
+                TrackedSummaryResponse response = new TrackedSummaryResponse(id, summary, dataNode, summaryNodes);
+                Message<TrackedSummaryResponse> message = Message.out(Verb.TRACKED_SUMMARY_RSP, response);
+                for (int node : remoteNodes)
+                    MessagingService.instance().send(message, host(node));
+            }
+
+            Iterator<Offsets> iter = summary.onlyUnreconciled();
+            Node2OffsetsMap missingOffsets = new Node2OffsetsMap();
+            while (iter.hasNext())
+            {
+                Offsets ourOffsets = iter.next();
+                CoordinatorLogId logId = ourOffsets.logId();
+                if (logId.hostId() != LOCAL_NODE)
+                    continue;
+
+                MutationTrackingService.instance.collectRemotelyMissingMutations(ourOffsets, remoteNodes, missingOffsets);
+                // we don't listen to or block on delivery of these mutations, intentionally
+                missingOffsets.forEach(ReadReconciliations::push);
+                missingOffsets.clear();
+            }
+
+            summaries.add(Pair.create(LOCAL_NODE, summary));
+            return updateRemainingAndMaybeComplete(0, -1, 0);
+        }
+
+        /**
+         * Calculate the diff with the remote summary to see which mutations from the summary we lack,
+         * and pull them - from the primary coordinator, if it's alive, or the remote replica, if it's not.
+         */
+        boolean acceptRemoteSummary(MutationSummary summary, int remoteNode)
+        {
+            Log2OffsetsMap.Mutable missingMutations = new Log2OffsetsMap.Mutable();
+            MutationTrackingService.instance.collectLocallyMissingMutations(summary, missingMutations);
+
+            // don't request what's already been requested for other remote summaries
+            // TODO (consider): rely entirely on IncomingMutations deduplication instead
+            missingMutations.removeAll(requested);
+            missingMutations.forEach(requested::add);
+            int missingCount = missingMutations.idCount();
+
+            // TODO (expected, low priority): handle timeouts here to stop earlier
+            IncomingMutations.Callback callback =
+                (mutationId) -> ReadReconciliations.instance.acceptMutation(id, mutationId);
+            for (Offsets offsets : missingMutations.offsets())
+                pull(remoteNode, offsets, callback);
+
+            summaries.add(Pair.create(remoteNode, summary));
+            return updateRemainingAndMaybeComplete(missingCount, -1, 0);
+        }
+
+        boolean acceptSyncAck(int ignoredSyncId)
+        {
+            return updateRemainingAndMaybeComplete(0, 0, -1);
+        }
+
+        boolean acceptMutation(ShortMutationId ignoredMutationId)
+        {
+            return updateRemainingAndMaybeComplete(-1, 0, 0);
+        }
+
+        /**
+         * Remote summaries minus data node summary offsets
+         */
+        private Log2OffsetsMap.Mutable augmentingOffsets()
+        {
+            Log2OffsetsMap.Mutable offsets = new Log2OffsetsMap.Mutable();
+
+            // add up all remote summaries
+            for (int i = 0, size = summaries.size(); i < size; i++)
+            {
+                Pair<Integer, MutationSummary> summary = summaries.get(i);
+                if (summary.left != LOCAL_NODE)
+                    offsets.addAll(summary.right);
+            }
+
+            // subtract the local summary
+            for (int i = 0, size = summaries.size(); i < size; i++)
+            {
+                Pair<Integer, MutationSummary> summary = summaries.get(i);
+                if (summary.left == LOCAL_NODE)
+                {
+                    offsets.removeAll(summary.right);
+                    break;
+                }
+            }
+
+            return offsets;
+        }
+
+        private boolean isDataNode()
+        {
+            return LOCAL_NODE == dataNode;
+        }
+
+        /*
+         * Logic to deal with remaining mutations/summaries/syncAcks counters atomically.
+         */
+
+        private static int remainingMutations(long remaining)
+        {
+            return (int) ((remaining >>> 32) & 0xFFFFFFFFL);
+        }
+
+        private static int remainingSummaries(long remaining)
+        {
+            return (int) ((remaining >>> 16) & 0xFFFFL);
+        }
+
+        private static int remainingSyncAcks(long remaining)
+        {
+            return (int) (remaining & 0xFFFFL);
+        }
+
+        // 32 bits for mutations, 16 bits for summaries, 16 bits for syncAcks
+        private static long remaining(int mutations, int summaries, int syncAcks)
+        {
+            return ((long) mutations << 32) | (((long) summaries << 16) | syncAcks);
+        }
+
+        private long updateRemaining(int mutationsDelta, int summariesDelta, int syncAcksDelta)
+        {
+            long prev, next;
+            do
+            {
+                prev = remaining;
+                int mutations = remainingMutations(prev) + mutationsDelta;
+                int summaries = remainingSummaries(prev) + summariesDelta;
+                int syncAcks = remainingSyncAcks(prev) + syncAcksDelta;
+                next = remaining(mutations, summaries, syncAcks);
+            } while (!remainingUpdater.compareAndSet(this, prev, next));
+            return next;
+        }
+
+        private boolean updateRemainingAndMaybeComplete(int mutationsDelta, int summariesDelta, int syncAcksDelta)
+        {
+            return updateRemaining(mutationsDelta, summariesDelta, syncAcksDelta) == 0 && complete();
+        }
+
+        private boolean complete()
+        {
+            if (isDataNode())
+                MutationTrackingService.instance.localReads().acknowledgeReconcile(id, augmentingOffsets());
+            else
+                MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_ACK, new ReadReconcileAck(id)), host(dataNode));
+
+            return true;
+        }
+
+        boolean isPurgeable(long nanoTime)
+        {
+            return nanoTime - expiresAtNanos > 0;
+        }
+    }
+
+    /**
+     * @param node node id of the remote replica from which we got the summary
+     * @param offsets offsets that we need to pull - from the coordinator, if alive, or from the
+     *                node we got the summary from (by definition it also has the mutation)
+     */
+    private static void pull(int node, Offsets offsets, IncomingMutations.Callback callback)
+    {
+        InetAddressAndPort logCoordinator = host(offsets.logId().hostId());
+        InetAddressAndPort pullFrom = FailureDetector.instance.isAlive(logCoordinator)
+                                    ? logCoordinator
+                                    : host(node);
+
+        Offsets.Mutable toPull = new Offsets.Mutable(offsets.logId());
+        for (ShortMutationId id : offsets)
+            if (MutationTrackingService.instance.registerMutationCallback(id, callback))
+                toPull.add(id.offset());
+
+        if (!toPull.isEmpty())
+        {
+            PullMutationsRequest pull = new PullMutationsRequest(Offsets.Immutable.copy(toPull));
+            MessagingService.instance().send(Message.out(Verb.PULL_MUTATIONS_REQ, pull), pullFrom);
+        }
+    }
+
+    private static void push(int node, Offsets offsets)
+    {
+        // TODO implement pre-emptive push with read priority
+    }
+
+    private static InetAddressAndPort host(int node)
+    {
+        return ClusterMetadata.current().directory.endpoint(new NodeId(node));
+    }
+
+    private static int node(InetAddressAndPort host)
+    {
+        return ClusterMetadata.current().directory.peerId(host).id();
+    }
+}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java
deleted file mode 100644
index c62531240a..0000000000
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java
+++ /dev/null
@@ -1,617 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.service.reads.tracked;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.cassandra.concurrent.Stage;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.ConsistencyLevel;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.ReadCommand;
-import org.apache.cassandra.db.ReadExecutionController;
-import org.apache.cassandra.db.filter.ColumnFilter;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.ReplicaPlan;
-import org.apache.cassandra.metrics.ReadRepairMetrics;
-import org.apache.cassandra.net.Message;
-import org.apache.cassandra.net.MessagingService;
-import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.replication.Log2OffsetsMap;
-import org.apache.cassandra.replication.MutationJournal;
-import org.apache.cassandra.replication.MutationSummary;
-import org.apache.cassandra.replication.ReconciliationPlan;
-import org.apache.cassandra.replication.ShortMutationId;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.AbstractFuture;
-import org.apache.cassandra.utils.concurrent.Accumulator;
-import org.apache.cassandra.utils.concurrent.AsyncPromise;
-import org.apache.cassandra.utils.concurrent.Future;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-
-public class TrackedLocalReadCoordinator
-{
-    private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReadCoordinator.class);
-
-    private final TrackedRead.Id readId;
-    private final AsyncPromise<TrackedDataResponse> promise;
-
-    private volatile State state;
-
-    public TrackedLocalReadCoordinator(TrackedRead.Id readId)
-    {
-        this.readId = readId;
-        this.promise = new AsyncPromise<>();
-        this.state = INITIALIZED;
-    }
-
-    public AbstractFuture<TrackedDataResponse> addCallback(BiConsumer<TrackedDataResponse, Throwable> callback)
-    {
-        return promise.addCallback(callback);
-    }
-
-    enum Status { INITIALIZED, AWAITING_READ, READING, RECONCILING, ABORTED, COMPLETED }
-
-    private static abstract class State
-    {
-        abstract Status status();
-
-        State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
-        {
-            // TODO: validate permitted state instead of just ignoring events?
-            return this;
-        }
-
-        State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
-        {
-            throw new IllegalStateException("Received in-progress read with status " + status());
-        }
-
-        State receiveSummary(InetAddressAndPort from, MutationSummary summary)
-        {
-            logger.trace("Ignoring summary from {} with status {}", from, status());
-            return this;
-        }
-
-        State acknowledgeSync(int syncId)
-        {
-            logger.trace("Ignoring sync ack {} with status {}", syncId, status());
-            return this;
-        }
-
-        State receiveMutations(List<Mutation> mutations)
-        {
-            logger.trace("Ignoring {} mutations with status {}", mutations.size(), status());
-            return this;
-        }
-
-        State abort()
-        {
-            return ABORTED;
-        }
-
-        boolean isPurgeable(long nanoTime)
-        {
-            return false;
-        }
-
-        boolean isReading()
-        {
-            return is(Status.READING);
-        }
-
-        boolean isCompleted()
-        {
-            return is(Status.COMPLETED);
-        }
-
-        boolean is(Status status)
-        {
-            return status() == status;
-        }
-
-        @Override
-        public String toString()
-        {
-            return status().toString();
-        }
-    }
-
-    static final State INITIALIZED = new State()
-    {
-        @Override
-        Status status() { return Status.INITIALIZED; }
-
-        @Override
-        State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
-        {
-            return new Reading(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos);
-        }
-
-        @Override
-        State receiveSummary(InetAddressAndPort from, MutationSummary summary)
-        {
-            return new AwaitingRead(from, summary);
-        }
-    };
-
-    static final State ABORTED = new State()
-    {
-        @Override
-        Status status() { return Status.ABORTED; }
-
-        @Override
-        State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
-        {
-            return this; // the read can't complete without data, but it could have been aborted in the meantime
-        }
-
-        @Override
-        boolean isPurgeable(long nanoTime) { return true; }
-    };
-
-    static final State COMPLETED = new State()
-    {
-        @Override
-        Status status() { return Status.COMPLETED; }
-
-        @Override
-        boolean isPurgeable(long nanoTime) { return true; }
-    };
-
-    private static class ReceivedSummary
-    {
-        final InetAddressAndPort from;
-        final MutationSummary summary;
-
-        ReceivedSummary(InetAddressAndPort from, MutationSummary summary)
-        {
-            this.from = from;
-            this.summary = summary;
-        }
-    }
-
-    // if we start receiving summaries before we receive the read command, they're
-    // collected here
-    private static class AwaitingRead extends State
-    {
-        private long lastUpdateNanos;
-        private final List<ReceivedSummary> summaries;
-
-        AwaitingRead(InetAddressAndPort summaryFrom, MutationSummary summary)
-        {
-             summaries = new ArrayList<>();
-             summaries.add(new ReceivedSummary(summaryFrom, summary));
-        }
-
-        @Override
-        Status status()
-        {
-            return Status.AWAITING_READ;
-        }
-
-        @Override
-        State startLocalRead(TrackedRead.Id readId, AsyncPromise<TrackedDataResponse> promise, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos)
-        {
-            return new Reading(readId, promise, command, replicaPlan, summaryNodes, summaries, expiresAtNanos);
-        }
-
-        @Override
-        State receiveSummary(InetAddressAndPort from, MutationSummary summary)
-        {
-            summaries.add(new ReceivedSummary(from, summary));
-            lastUpdateNanos = Clock.Global.nanoTime();
-            return this;
-        }
-
-        @Override
-        boolean isPurgeable(long nanoTime)
-        {
-            return nanoTime - lastUpdateNanos > DatabaseDescriptor.getReadRpcTimeout(TimeUnit.NANOSECONDS);
-        }
-    }
-
-    private static class Reading extends State
-    {
-        private final TrackedRead.Id readId;
-        private final AsyncPromise<TrackedDataResponse> promise;
-        private final ReadCommand command;
-        private volatile PartialTrackedRead read;
-        private final long expiresAtNanos;
-        private final ReplicaPlan.AbstractForRead<?, ?> replicaPlan;
-        private final Accumulator<ReceivedSummary> summaries;
-        private final int[] summaryNodes; // for speculating when we haven't received enough summaries
-
-        Reading(
-            TrackedRead.Id readId,
-            AsyncPromise<TrackedDataResponse> promise,
-            ReadCommand command,
-            ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
-            int[] summaryNodes,
-            long expiresAtNanos)
-        {
-            this.readId = readId;
-            this.promise = promise;
-            this.expiresAtNanos = expiresAtNanos;
-            this.command = command;
-            this.replicaPlan = replicaPlan;
-            this.summaries = new Accumulator<>(replicaPlan.readCandidates().size());
-            this.summaryNodes = summaryNodes;
-        }
-
-        Reading(
-            TrackedRead.Id readId,
-            AsyncPromise<TrackedDataResponse> promise,
-            ReadCommand command,
-            ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
-            int[] summaryNodes,
-            List<ReceivedSummary> summaries,
-            long expiresAtNanos)
-        {
-            this(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos);
-            for (ReceivedSummary summary : summaries) this.summaries.add(summary);
-        }
-
-        @Override
-        Status status()
-        {
-            return Status.READING;
-        }
-
-        @Override
-        boolean isPurgeable(long nanoTime)
-        {
-            return nanoTime - expiresAtNanos > 0;
-        }
-
-        private State maybeComplete()
-        {
-            if (read == null || summaries.size() < replicaPlan.readQuorum())
-                return this;
-
-            Map<InetAddressAndPort, MutationSummary> summaryMap = new HashMap<>();
-            summaries.snapshot().forEach(rc -> summaryMap.put(rc.from, rc.summary));
-
-            Map<InetAddressAndPort, ReconciliationPlan> reconciliations = ReconciliationPlan.calculateReconciliation(summaryMap);
-
-            if (reconciliations.isEmpty())
-            {
-                logger.trace("Read complete for {}", readId);
-                complete(promise, read, command.columnFilter(), replicaPlan.consistencyLevel(), expiresAtNanos);
-                return COMPLETED;
-            }
-            else
-            {
-                logger.trace("Beginning reconciliation for {}", readId);
-                Reconciling reconciling = new Reconciling(readId, promise, command, read, replicaPlan.consistencyLevel(), expiresAtNanos, reconciliations);
-                reconciling.start();  // TODO: don't do this until after the coordinator state is set to reconciling if converting to lock free
-                return reconciling;
-            }
-        }
-
-        @Override
-        State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary)
-        {
-            if (this.read != null)
-                return this;
-
-            logger.trace("In progress read received for {}", readId);
-            this.read = read;
-            summaries.add(new ReceivedSummary(FBUtilities.getBroadcastAddressAndPort(), summary));
-
-            return maybeComplete();
-        }
-
-        @Override
-        State receiveSummary(InetAddressAndPort from, MutationSummary summary)
-        {
-            logger.trace("Summary received from {} for {}", from, readId);
-            summaries.add(new ReceivedSummary(from, summary));
-            return maybeComplete();
-        }
-
-        @Override
-        State abort()
-        {
-            logger.trace("Aborting read {}", readId);
-            if (read != null)
-                read.close();
-            return ABORTED;
-        }
-    }
-
-    private static class PendingSync
-    {
-        final int syncId;
-        final InetAddressAndPort from;
-        final InetAddressAndPort to;
-        final Log2OffsetsMap.Immutable plan;
-
-        PendingSync(int syncId, InetAddressAndPort from, InetAddressAndPort to, Log2OffsetsMap.Immutable plan)
-        {
-            this.syncId = syncId;
-            this.from = from;
-            this.to = to;
-            this.plan = plan;
-        }
-
-        ReadReconcileSend.PeerSync toPeerSync()
-        {
-            return new ReadReconcileSend.PeerSync(syncId, to, plan);
-        }
-    }
-
-    private static class Reconciling extends State
-    {
-        private final TrackedRead.Id readId;
-        private final AsyncPromise<TrackedDataResponse> promise;
-        private final ReadCommand command;
-        private final PartialTrackedRead read;
-        private final ConsistencyLevel consistencyLevel;
-        private final long expiresAtNanos;
-
-        final Map<InetAddressAndPort, ReconciliationPlan> plans;
-        final Log2OffsetsMap.Mutable outstandingMutations = new Log2OffsetsMap.Mutable();
-        final Map<Integer, PendingSync> pendingSync = new ConcurrentHashMap<>();
-        final int blockFor;
-
-        Reconciling(
-            TrackedRead.Id readId,
-            AsyncPromise<TrackedDataResponse> promise,
-            ReadCommand command,
-            PartialTrackedRead read,
-            ConsistencyLevel consistencyLevel,
-            long expiresAtNanos,
-            Map<InetAddressAndPort, ReconciliationPlan> plans)
-        {
-            this.readId = readId;
-            this.promise = promise;
-            this.command = command;
-            this.read = Preconditions.checkNotNull(read);
-            this.consistencyLevel = consistencyLevel;
-            this.expiresAtNanos = expiresAtNanos;
-            this.plans = plans;
-
-            int syncs = 0;
-            int nextSyncId = 0;
-            for (Map.Entry<InetAddressAndPort, ReconciliationPlan> entry : plans.entrySet())
-            {
-                InetAddressAndPort from = entry.getKey();
-                ReconciliationPlan plan = entry.getValue();
-                for (InetAddressAndPort to : plan.nodes())
-                {
-                    int syncId = nextSyncId++;
-                    PendingSync sync = new PendingSync(syncId, from, to, plan.peerReconciliation(to));
-                    pendingSync.put(syncId, sync);
-                    syncs++;
-                    if (to.equals(FBUtilities.getBroadcastAddressAndPort()))
-                    {
-                        outstandingMutations.addAll(plan.offsetsFor(to));
-                    }
-                }
-            }
-
-            if (logger.isTraceEnabled())
-                logger.trace("Reconciling {} syncs, {} mutations for {}", syncs, outstandingMutations.idCount(), readId);
-
-            this.blockFor = syncs;
-        }
-
-        @Override
-        Status status()
-        {
-            return Status.RECONCILING;
-        }
-
-        @Override
-        boolean isPurgeable(long nanoTime)
-        {
-            return nanoTime - expiresAtNanos > 0;
-        }
-
-        void start()
-        {
-            ReadRepairMetrics.trackedReconcile.mark();
-            ColumnFamilyStore.metricsFor(command.metadata().id).readRepairRequests.mark();
-
-            Map<InetAddressAndPort, List<ReadReconcileSend.PeerSync>> peerSync = new HashMap<>();
-            pendingSync.values().forEach(pending ->
-                peerSync.computeIfAbsent(pending.from, node -> new ArrayList<>()).add(pending.toPeerSync()));
-
-            for (Map.Entry<InetAddressAndPort, List<ReadReconcileSend.PeerSync>> entry : peerSync.entrySet())
-            {
-                Message<ReadReconcileSend> message = Message.out(Verb.READ_RECONCILE_SEND, new ReadReconcileSend(readId, entry.getValue()));
-                logger.trace("Sending read reconciliation for {} {} to {}", readId, message.payload, entry.getKey());
-                MessagingService.instance().send(message, entry.getKey());
-            }
-        }
-
-        private State maybeComplete()
-        {
-            if (!pendingSync.isEmpty() || !outstandingMutations.isEmpty())
-                return this;
-
-            logger.trace("Reconciliation completed for read {}", readId);
-            complete(promise, read, command.columnFilter(), consistencyLevel, expiresAtNanos);
-            return COMPLETED;
-        }
-
-        @Override
-        State acknowledgeSync(int syncId)
-        {
-            logger.trace("Reconciliation sync {} received for {}", syncId, readId);
-            pendingSync.remove(syncId);
-            return maybeComplete();
-        }
-
-        @Override
-        State receiveMutations(List<Mutation> mutations)
-        {
-            Log2OffsetsMap.Mutable received = new Log2OffsetsMap.Mutable();
-            mutations.forEach(mutation -> {
-                logger.trace("Received mutation {} for read {}", mutation.id(), readId);
-                received.add(mutation.id());
-            });
-            outstandingMutations.removeAll(received);
-
-            if (logger.isTraceEnabled())
-                logger.trace("Received {} mutations, {} mutations outstanding for {}", mutations.size(), outstandingMutations.idCount(), readId);
-            read.augment(mutations);
-            return maybeComplete();
-        }
-
-        @Override
-        State abort()
-        {
-            read.close();
-            return ABORTED;
-        }
-    }
-
-    @Override
-    public String toString()
-    {
-        return "TrackedLocalReadCoordinator{" + readId + ':' + state.status() + '}';
-    }
-
-    @VisibleForTesting
-    public static void processDelta(PartialTrackedRead read, MutationSummary initialSummary, MutationSummary secondarySummary)
-    {
-        // Compute any mutations that we could've missed during initial read execution.
-        ArrayList<ShortMutationId> delta = new ArrayList<>();
-        MutationSummary.difference(secondarySummary, initialSummary, delta);
-
-        delta.forEach(mutationId -> {
-            Mutation mutation = MutationJournal.instance.read(mutationId);
-            Preconditions.checkNotNull(mutation);
-            read.augment(mutation);
-        });
-    }
-
-    public void startLocalRead(TrackedRead.Id readId, ReadCommand command, ReplicaPlan.AbstractForRead<?, ?> replicaPlan, int[] summaryNodes, long expiresAtNanos, Consumer<PartialTrackedRead> partialReadConsumer)
-    {
-        synchronized (this)
-        {
-            if (!(state = state.startLocalRead(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos)).isReading())
-                return;
-        }
-
-        PartialTrackedRead read;
-        MutationSummary secondarySummary;
-
-        MutationSummary initialSummary = command.createMutationSummary(false);
-        ReadExecutionController controller = command.executionController(false);
-        try
-        {
-            read = command.beginTrackedRead(controller);
-            if (partialReadConsumer != null)
-                partialReadConsumer.accept(read);
-            // Create another summary once initial data has been read fully. We do this to catch
-            // any mutations that may have arrived during initial read execution.
-            secondarySummary = command.createMutationSummary(true);
-            processDelta(read, initialSummary, secondarySummary);
-        }
-        catch (Exception e)
-        {
-            controller.close();
-            abort();
-            throw e;
-        }
-
-        synchronized (this)
-        {
-            state = state.receiveInProgressRead(read, secondarySummary);
-        }
-    }
-
-    private static void complete(AsyncPromise<TrackedDataResponse> promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos)
-    {
-        Stage.READ.submit(() -> completeInternal(promise, read, selection, consistencyLevel, expiresAtNanos));
-    }
-
-    private static void completeInternal(AsyncPromise<TrackedDataResponse> promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos)
-    {
-        try (PartialTrackedRead.CompletedRead completedRead = read.complete())
-        {
-            TrackedDataResponse response = completedRead.response();
-            Future<TrackedDataResponse> followUp = completedRead.followupRead(response, consistencyLevel, expiresAtNanos);
-
-            if (followUp != null)
-            {
-                followUp.addCallback((newResponse, error) -> {
-                    if (error != null)
-                    {
-                        promise.tryFailure(error);
-                        return;
-                    }
-                    promise.trySuccess(newResponse);
-                });
-            }
-            else
-            {
-                promise.trySuccess(response);
-            }
-        }
-        catch (Exception e)
-        {
-            promise.tryFailure(e);
-            throw e;
-        }
-        finally
-        {
-            read.close();
-        }
-    }
-
-    synchronized void receiveSummary(InetAddressAndPort from, MutationSummary summary)
-    {
-        if (logger.isTraceEnabled())
-            logger.trace("Received summary {} from {}, for {}", summary, from, state);
-        state = state.receiveSummary(from, summary);
-    }
-
-    synchronized boolean acknowledgeSync(int syncId)
-    {
-        return (state = state.acknowledgeSync(syncId)).isCompleted();
-    }
-
-    synchronized boolean receiveMutations(List<Mutation> mutations)
-    {
-        return (state = state.receiveMutations(mutations)).isCompleted();
-    }
-
-    synchronized void abort()
-    {
-        state = state.abort();
-    }
-
-    boolean isTimedOutOrComplete(long nanoTime)
-    {
-        return state.isPurgeable(nanoTime);
-    }
-}
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
index 24bc0b5260..4f8bca8f25 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java
@@ -15,88 +15,69 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.service.reads.tracked;
 
-import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
-import org.apache.cassandra.concurrent.Shutdownable;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import java.util.ArrayList;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.concurrent.Stage;
 import org.apache.cassandra.db.*;
-import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
 import org.apache.cassandra.service.reads.ReadCoordinator;
+import org.apache.cassandra.metrics.ReadRepairMetrics;
+import org.apache.cassandra.replication.ExpiredStatePurger;
+import org.apache.cassandra.replication.Log2OffsetsMap;
+import org.apache.cassandra.replication.MutationJournal;
+import org.apache.cassandra.replication.MutationSummary;
+import org.apache.cassandra.replication.ShortMutationId;
 import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
 import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.concurrent.AsyncPromise;
+import org.apache.cassandra.utils.concurrent.Future;
+import org.jctools.maps.NonBlockingHashMap;
+
+import org.apache.cassandra.transport.Dispatcher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
-import static java.util.concurrent.TimeUnit.MINUTES;
-import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
-import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL;
-
 /**
- * Since the read reconciliations don't use 2 way callbacks, a map of active reconciliations
+ * Since the read reconciliations don't use 2 way callbacks, maps of active reads and reconciliations
  * are maintained and expired here.
- *
+ * <p>
  * Borrowed heavily from RequestCallbacks
  */
-public class TrackedLocalReads implements Shutdownable
+public class TrackedLocalReads implements ExpiredStatePurger.Expireable
 {
     private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReads.class);
 
-    private final ConcurrentMap<TrackedRead.Id, TrackedLocalReadCoordinator> reads = new ConcurrentHashMap<>();
-    private final ScheduledExecutorPlus executor = executorFactory().scheduled("Reconciliation-Map-Reaper", NORMAL);
+    private final NonBlockingHashMap<TrackedRead.Id, Coordinator> coordinators = new NonBlockingHashMap<>();
 
     public TrackedLocalReads()
     {
-        long expirationInterval = defaultExpirationInterval();
-        executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS);
+        ExpiredStatePurger.instance.register(this);
     }
 
-    private void expire()
-    {
-        long start = Clock.Global.nanoTime();
-        int n = 0;
-        for (Map.Entry<TrackedRead.Id, TrackedLocalReadCoordinator> entry : reads.entrySet())
-        {
-            TrackedLocalReadCoordinator read = entry.getValue();
-            if (read.isTimedOutOrComplete(start))
-            {
-                read.abort();
-                if (reads.remove(entry.getKey(), entry.getValue()))
-                    n++;
-            }
-        }
-        if (n > 0)
-            logger.trace("Expired {} entries", n);
-    }
-
-    private TrackedLocalReadCoordinator getOrCreate(TrackedRead.Id id)
-    {
-        return reads.computeIfAbsent(id, TrackedLocalReadCoordinator::new);
-    }
-
-    public void receiveSummary(InetAddressAndPort from, TrackedSummaryResponse summary)
-    {
-        getOrCreate(summary.readId()).receiveSummary(from, summary.summary());
-    }
-
-    public TrackedLocalReadCoordinator beginRead(TrackedRead.Id readId, ClusterMetadata metadata, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes, long expiresAtNanos, Consumer<PartialTrackedRead> partialReadConsumer)
+    public AsyncPromise<TrackedDataResponse> beginRead(
+        TrackedRead.Id readId,
+        ClusterMetadata metadata,
+        ReadCommand command,
+        ConsistencyLevel consistencyLevel,
+        int[] summaryNodes,
+        Dispatcher.RequestTime requestTime,
+        Consumer<PartialTrackedRead> partialReadConsumer)
     {
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
         SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry;
         ReplicaPlan.AbstractForRead<?, ?> replicaPlan;
+
         if (command instanceof SinglePartitionReadCommand)
         {
             replicaPlan = ReplicaPlans.forRead(metadata,
@@ -119,69 +100,170 @@ public class TrackedLocalReads implements Shutdownable
                                                     1);
         }
         // TODO: confirm all summaryNodes are present in the replica plan
-
-        TrackedLocalReadCoordinator coordinator = getOrCreate(readId);
-        coordinator.startLocalRead(readId, command, replicaPlan, summaryNodes, expiresAtNanos, partialReadConsumer);
-        return coordinator;
+        AsyncPromise<TrackedDataResponse> promise = new AsyncPromise<>();
+        beginReadInternal(readId, command, replicaPlan, summaryNodes, requestTime, partialReadConsumer, promise);
+        return promise;
     }
 
-    public void acknowledgeSync(TrackedRead.Id readId, int syncId)
+    // TODO (expected): skip local summaries and reconcile when summaryNodes is empty (e.g. for CL.ONE)
+    private void beginReadInternal(
+        TrackedRead.Id readId,
+        ReadCommand command,
+        ReplicaPlan.AbstractForRead<?, ?> replicaPlan,
+        int[] summaryNodes,
+        Dispatcher.RequestTime requestTime,
+        Consumer<PartialTrackedRead> partialReadConsumer,
+        AsyncPromise<TrackedDataResponse> promise)
     {
-        TrackedLocalReadCoordinator read = reads.get(readId);
-        if (read == null)
-            return;
+        PartialTrackedRead read = null;
+        MutationSummary secondarySummary;
 
-        if (read.acknowledgeSync(syncId))
-            reads.remove(readId);
-    }
+        MutationSummary initialSummary = command.createMutationSummary(false);
+        ReadExecutionController controller = command.executionController(false);
 
-    public boolean receiveMutations(TrackedRead.Id readId, int syncId, List<Mutation> mutations)
-    {
-        TrackedLocalReadCoordinator read = reads.get(readId);
-        if (read == null)
-            return false;
-
-        read.receiveMutations(mutations);
-        if (read.acknowledgeSync(syncId))
-            reads.remove(readId);
-        return true;
-    }
+        try
+        {
+            read = command.beginTrackedRead(controller);
+            // Create another summary once initial data has been read fully. We do this to catch
+            // any mutations that may have arrived during initial read execution.
+            secondarySummary = command.createMutationSummary(true);
+            processDelta(read, initialSummary, secondarySummary);
+        }
+        catch (Exception e)
+        {
+            controller.close();
+            logger.trace("Aborting read {}", readId);
+            if (read != null) read.close();
+            throw e;
+        }
 
-    public static long defaultExpirationInterval()
-    {
-        return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2;
+        Coordinator coordinator =
+                new Coordinator(readId, promise, command.columnFilter(), read, replicaPlan.consistencyLevel(), requestTime);
+        coordinators.put(readId, coordinator);
+
+        // TODO (expected): reconsider the approach to tracked mutation metrics
+        ReadRepairMetrics.trackedReconcile.mark();
+
+        // TODO (consider): is it possible to exit right here if this is the last missing summary, and we can tell that
+        //                  no node needs anything from anyone?
+
+        // pass the final summary to the reconcile service
+        ReadReconciliations.instance.acceptLocalSummary(readId, secondarySummary, summaryNodes);
     }
 
-    @Override
-    public void shutdown()
+    @VisibleForTesting
+    public static void processDelta(PartialTrackedRead read, MutationSummary initialSummary, MutationSummary secondarySummary)
     {
-        executor.shutdown();
+        // Compute any mutations that we could've missed during initial read execution.
+        ArrayList<ShortMutationId> delta = new ArrayList<>();
+        MutationSummary.difference(secondarySummary, initialSummary, delta);
+
+        delta.forEach(mutationId -> {
+            Mutation mutation = MutationJournal.instance.read(mutationId);
+            Preconditions.checkNotNull(mutation);
+            read.augment(mutation);
+        });
     }
 
-    @Override
-    public boolean isTerminated()
+    public void acknowledgeReconcile(TrackedRead.Id readId, Log2OffsetsMap<?> augmentingOffsets)
     {
-        return executor.isTerminated();
+        Coordinator coordinator = coordinators.remove(readId);
+        if (coordinator != null)
+            coordinator.acknowledgeReconcile(augmentingOffsets);
     }
 
     @Override
-    public Object shutdownNow()
+    public int expire(long nanoTime)
     {
-        return executor.shutdownNow();
+        int n = 0;
+        for (Map.Entry<TrackedRead.Id, Coordinator> entry : coordinators.entrySet())
+        {
+            TrackedRead.Id id = entry.getKey();
+            Coordinator coordinator = entry.getValue();
+            if (coordinator.isPurgeable(nanoTime) && coordinators.remove(id, coordinator))
+            {
+                coordinator.abort();
+                n++;
+            }
+        }
+        return n;
     }
 
-    public void shutdownBlocking() throws InterruptedException
+    private static class Coordinator
     {
-        if (executor == null || executor.isTerminated())
-            return;
+        private final TrackedRead.Id readId;
+        private final AsyncPromise<TrackedDataResponse> promise;
+        private final ColumnFilter selection;
+        private final PartialTrackedRead read;
+        private final ConsistencyLevel consistencyLevel;
+        private final Dispatcher.RequestTime requestTime;
 
-        executor.shutdown();
-        executor.awaitTermination(1, MINUTES);
-    }
+        Coordinator(
+                TrackedRead.Id readId,
+                AsyncPromise<TrackedDataResponse> promise,
+                ColumnFilter selection,
+                PartialTrackedRead read,
+                ConsistencyLevel consistencyLevel,
+                Dispatcher.RequestTime requestTime)
+        {
+            this.readId = readId;
+            this.promise = promise;
+            this.selection = selection;
+            this.read = Preconditions.checkNotNull(read);
+            this.consistencyLevel = consistencyLevel;
+            this.requestTime = requestTime;
+        }
 
-    @Override
-    public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException
-    {
-        return executor.awaitTermination(timeout, units);
+        boolean isPurgeable(long nanoTime)
+        {
+            long deadline = requestTime.computeDeadline(read.command().verb().expiresAfterNanos());
+            return nanoTime - deadline > 0;
+        }
+
+        void abort()
+        {
+            read.close();
+        }
+
+        void acknowledgeReconcile(Log2OffsetsMap<?> augmentingOffsets)
+        {
+            logger.trace("Reconciliation completed for {}, missing {}", readId, augmentingOffsets);
+            Stage.READ.submit(() -> { read.augment(augmentingOffsets); complete(); });
+        }
+
+        private void complete()
+        {
+            try (PartialTrackedRead.CompletedRead completedRead = read.complete())
+            {
+                TrackedDataResponse response = completedRead.response();
+                Future<TrackedDataResponse> followUp = completedRead.followupRead(response, consistencyLevel, requestTime);
+
+                if (followUp != null)
+                {
+                    ReadCommand command = read.command();
+                    followUp.addCallback((newResponse, error) -> {
+                        if (error != null)
+                        {
+                            promise.tryFailure(error);
+                            return;
+                        }
+                        promise.trySuccess(newResponse);
+                    });
+                }
+                else
+                {
+                    promise.trySuccess(response);
+                }
+            }
+            catch (Exception e)
+            {
+                promise.tryFailure(e);
+                throw e;
+            }
+            finally
+            {
+                read.close();
+            }
+        }
     }
 }
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
index 0d029a554c..267670a63e 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java
@@ -29,6 +29,8 @@ import org.apache.cassandra.db.rows.RowIterator;
 import org.apache.cassandra.exceptions.ReadFailureException;
 import org.apache.cassandra.exceptions.ReadTimeoutException;
 import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.exceptions.UnavailableException;
+import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -58,6 +60,7 @@ import java.util.function.Consumer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
 
 public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>> implements RequestCallback<TrackedDataResponse>
@@ -140,6 +143,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
     private final ReadCommand command;
     private final ReplicaPlan.AbstractForRead<E, P> replicaPlan;
     private final ConsistencyLevel consistencyLevel;
+    private final Dispatcher.RequestTime requestTime;
 
     private static class RequestFailure extends Throwable
     {
@@ -158,11 +162,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
         }
     }
 
-    public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead<E, P> replicaPlan, ConsistencyLevel consistencyLevel)
+    public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead<E, P> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
     {
         this.command = command;
         this.replicaPlan = replicaPlan;
         this.consistencyLevel = consistencyLevel;
+        this.requestTime = requestTime;
     }
 
     @Override
@@ -180,12 +185,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
 
     public static class Partition extends TrackedRead<EndpointsForToken, ReplicaPlan.ForTokenRead>
     {
-        private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForToken, ReplicaPlan.ForTokenRead> replicaPlan, ConsistencyLevel consistencyLevel)
+        private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForToken, ReplicaPlan.ForTokenRead> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
         {
-            super(command, replicaPlan, consistencyLevel);
+            super(command, replicaPlan, consistencyLevel, requestTime);
         }
 
-        public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel)
+        public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
         {
             Preconditions.checkArgument(command.metadata().replicationType().isTracked());
             Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
@@ -199,7 +204,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
                                                                         consistencyLevel,
                                                                         retry,
                                                                         ReadCoordinator.DEFAULT);
-            return new Partition(command, replicaPlan, consistencyLevel);
+            return new Partition(command, replicaPlan, consistencyLevel, requestTime);
         }
 
         @Override
@@ -211,15 +216,15 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
 
     public static class Range extends TrackedRead<EndpointsForRange, ReplicaPlan.ForRangeRead>
     {
-        private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForRange, ReplicaPlan.ForRangeRead> replicaPlan, ConsistencyLevel consistencyLevel)
+        private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead<EndpointsForRange, ReplicaPlan.ForRangeRead> replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime)
         {
-            super(command, replicaPlan, consistencyLevel);
+            super(command, replicaPlan, consistencyLevel, requestTime);
         }
 
-        public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan)
+        public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan, Dispatcher.RequestTime requestTime)
         {
             Preconditions.checkArgument(command.metadata().replicationType().isTracked());
-            return new Range(command, replicaPlan, replicaPlan.consistencyLevel());
+            return new Range(command, replicaPlan, replicaPlan.consistencyLevel(), requestTime);
         }
 
         @Override
@@ -239,7 +244,13 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
         return hostids;
     }
 
-    private void start(long expiresAt, Consumer<PartialTrackedRead> partialReadConsumer)
+    private static int endpointToHostId(Replica replica)
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        return metadata.directory.peerId(replica.endpoint()).id();
+    }
+
+    private void start(Dispatcher.RequestTime requestTime, Consumer<PartialTrackedRead> partialReadConsumer)
     {
         // TODO: skip local coordination if this node knows its recovering from an outage
         // TODO: read speculation
@@ -252,20 +263,25 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
         // create an id
         // select data node
         // select summary nodes
-        E selected = replicaPlan.contacts();
-        Replica dataNode = localReplica != null && localReplica.isFull()
-                           ? localReplica
-                           : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1));
-        E summaryNodes = selected.filter(r -> r != dataNode);
-        int[] summaryHostIds = endpointsToHostIds(summaryNodes);
-
-
-        if (dataNode == localReplica)
+        E selected = replicaPlan.contacts().filter(r -> FailureDetector.instance.isAlive(r.endpoint()));
+        if (selected.size() < replicaPlan.readQuorum())
+            throw new UnavailableException(String.format("Insufficient replicas available for read (%d < %d)", selected.size(), replicaPlan.readQuorum()),
+                                           replicaPlan.consistencyLevel(), selected.size(), replicaPlan.readQuorum());
+        Replica dataReplica = localReplica != null && localReplica.isFull()
+                            ? localReplica
+                            : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1));
+        E summaryReplicas = selected.filter(r -> r != dataReplica);
+
+        int dataNode = endpointToHostId(dataReplica);
+        int[] summaryNodes = endpointsToHostIds(summaryReplicas);
+
+        if (dataReplica == localReplica)
         {
             logger.trace("Locally coordinating {}", readId);
             Stage.READ.submit(() -> {
-                TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, ClusterMetadata.current(), command, consistencyLevel, summaryHostIds, expiresAt, partialReadConsumer);
-                coordinator.addCallback((response, error) -> {
+                AsyncPromise<TrackedDataResponse> promise =
+                    MutationTrackingService.instance.localReads().beginRead(readId, ClusterMetadata.current(), command, consistencyLevel, summaryNodes, requestTime, partialReadConsumer);
+                promise.addCallback((response, error) -> {
                     if (error != null)
                     {
                         // TODO: notify coordinator that read has failed
@@ -279,22 +295,22 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
         }
         else
         {
-            logger.trace("Sending data request for {} to {}", readId, dataNode.endpoint());
+            logger.trace("Sending data request for {} to {}", readId, dataReplica.endpoint());
             Preconditions.checkArgument(partialReadConsumer == null, "Cannot supply read consumer for nonlocal reads");
-            DataRequest dataRequest = new DataRequest(readId, command, consistencyLevel, summaryHostIds);
-            Message<DataRequest> dataMessage = Message.outWithFlag(verb(), dataRequest, MessageFlag.CALL_BACK_ON_FAILURE);
-            MessagingService.instance().sendWithCallback(dataMessage, dataNode.endpoint(), this);
+            DataRequest dataRequest = new DataRequest(readId, command, dataNode, summaryNodes, consistencyLevel);
+            Message<DataRequest> dataMessage = Message.builder(verb(), dataRequest)
+                                                      .withRequestTime(requestTime)
+                                                      .withFlag(MessageFlag.CALL_BACK_ON_FAILURE)
+                                                      .build();
+            MessagingService.instance().sendWithCallback(dataMessage, dataReplica.endpoint(), this);
         }
 
-        if (summaryNodes.isEmpty())
+        if (summaryReplicas.isEmpty())
             return;
 
-        SummaryRequest summaryRequest = new SummaryRequest(readId, command);
-        Message<SummaryRequest> summaryMessage = Message.outWithParam(Verb.TRACKED_SUMMARY_REQ,
-                                                                      summaryRequest,
-                                                                      ParamType.RESPOND_TO,
-                                                                      dataNode.endpoint());
-        for (Replica replica : summaryNodes)
+        SummaryRequest summaryRequest = new SummaryRequest(readId, command, dataNode, summaryNodes);
+        Message<SummaryRequest> summaryMessage = Message.outWithRequestTime(Verb.TRACKED_SUMMARY_REQ, summaryRequest, requestTime);
+        for (Replica replica : summaryReplicas)
         {
             if (localReplica == replica)
             {
@@ -309,19 +325,14 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
         }
     }
 
-    public void start(long expiresAt)
-    {
-        start(expiresAt, null);
-    }
-
-    public void startLocal(long expiresAt, Consumer<PartialTrackedRead> partialReadConsumer)
+    public void start(Dispatcher.RequestTime requestTime)
     {
-        start(expiresAt, partialReadConsumer);
+        start(requestTime, null);
     }
 
-    public void start(Dispatcher.RequestTime requestTime)
+    public void startLocal(Dispatcher.RequestTime requestTime, Consumer<PartialTrackedRead> partialReadConsumer)
     {
-        start(requestTime.computeDeadline(verb().expiresAfterNanos()));
+        start(requestTime, partialReadConsumer);
     }
 
     private void onResponse(TrackedDataResponse response)
@@ -356,7 +367,7 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
     {
         try
         {
-            return future.get(command.getTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS).makeIterator(command);
+            return future.get(Math.max(0, requestTime.computeDeadline(verb().expiresAfterNanos()) - Clock.Global.nanoTime()), NANOSECONDS).makeIterator(command);
         }
         catch (InterruptedException e)
         {
@@ -407,13 +418,17 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
 
     public abstract static class Request
     {
-        protected final Id readId;
-        protected final ReadCommand command;
+        public final Id readId;
+        public final ReadCommand command;
+        public final int dataNode;
+        public final int[] summaryNodes;
 
-        protected Request(Id readId, ReadCommand command)
+        protected Request(Id readId, ReadCommand command, int dataNode, int[] summaryNodes)
         {
             this.readId = readId;
             this.command = command;
+            this.dataNode = dataNode;
+            this.summaryNodes = summaryNodes;
         }
 
         public abstract void executeLocally(Message<? extends Request> message, ClusterMetadata metadata);
@@ -422,20 +437,22 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
     public static class DataRequest extends Request
     {
         private final ConsistencyLevel consistencyLevel;
-        private final int[] summaryNodes;
 
-        public DataRequest(Id readId, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes)
+        public DataRequest(Id readId, ReadCommand command, int dataNode, int[] summaryNodes, ConsistencyLevel consistencyLevel)
         {
-            super(readId, command);
+            super(readId, command, dataNode, summaryNodes);
             this.consistencyLevel = consistencyLevel;
-            this.summaryNodes = summaryNodes;
         }
 
         @Override
         public void executeLocally(Message<? extends Request> message, ClusterMetadata metadata)
         {
-            TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, metadata, command, consistencyLevel, summaryNodes, message.expiresAtNanos(), null);
-            coordinator.addCallback((response, error) -> {
+            Dispatcher.RequestTime requestTime = new Dispatcher.RequestTime(message.createdAtNanos());
+            AsyncPromise<TrackedDataResponse> promise =
+                MutationTrackingService.instance
+                                       .localReads()
+                                       .beginRead(readId, metadata, command, consistencyLevel, summaryNodes, requestTime, null);
+            promise.addCallback((response, error) -> {
                 if (error != null)
                 {
                     // TODO: notify coordinator that read has failed
@@ -453,10 +470,11 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
             {
                 Id.serializer.serialize(request.readId, out, version);
                 ReadCommand.serializer.serialize(request.command, out, version);
-                out.writeInt(request.consistencyLevel.code);
+                out.writeInt(request.dataNode);
                 out.writeInt(request.summaryNodes.length);
                 for (int hostid : request.summaryNodes)
                     out.writeInt(hostid);
+                out.writeInt(request.consistencyLevel.code);
             }
 
             @Override
@@ -464,11 +482,12 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
             {
                 Id readId = Id.serializer.deserialize(in, version);
                 ReadCommand command = ReadCommand.serializer.deserialize(in, version);
-                ConsistencyLevel consistencyLevel = ConsistencyLevel.fromCode(in.readInt());
+                int dataNode = in.readInt();
                 int[] summaryNodes = new int[in.readInt()];
                 for (int i = 0; i < summaryNodes.length; i++)
                     summaryNodes[i] = in.readInt();
-                return new DataRequest(readId, command, consistencyLevel, summaryNodes);
+                ConsistencyLevel consistencyLevel = ConsistencyLevel.fromCode(in.readInt());
+                return new DataRequest(readId, command, dataNode, summaryNodes, consistencyLevel);
             }
 
             @Override
@@ -476,26 +495,30 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
             {
                 return Id.serializer.serializedSize(request.readId, version) +
                        ReadCommand.serializer.serializedSize(request.command, version) +
-                       TypeSizes.sizeof(request.consistencyLevel.code) +
+                       TypeSizes.sizeof(request.dataNode) +
                        TypeSizes.sizeof(request.summaryNodes.length) +
-                       ((long) TypeSizes.INT_SIZE * request.summaryNodes.length);
+                       ((long) TypeSizes.INT_SIZE * request.summaryNodes.length) +
+                       TypeSizes.sizeof(request.consistencyLevel.code);
             }
         };
     }
 
     public static class SummaryRequest extends Request
     {
-        public SummaryRequest(Id readId, ReadCommand command)
+        public SummaryRequest(Id readId, ReadCommand command, int dataNode, int[] summaryNodes)
         {
-            super(readId, command);
+            super(readId, command, dataNode, summaryNodes);
         }
 
         @Override
         public void executeLocally(Message<? extends Request> message, ClusterMetadata metadata)
         {
-            MutationSummary summary = command.createMutationSummary(false);
-            TrackedSummaryResponse response = new TrackedSummaryResponse(readId, summary);
-            MessagingService.instance().send(message.responseWith(response), message.respondTo());
+            ReadReconciliations.instance.handleSummaryRequest((SummaryRequest) message.payload);
+        }
+
+        public TrackedSummaryResponse makeResponse(MutationSummary summary)
+        {
+            return new TrackedSummaryResponse(readId, summary, dataNode, summaryNodes);
         }
 
         public static final IVersionedSerializer<SummaryRequest> serializer = new IVersionedSerializer<>()
@@ -505,6 +528,10 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
             {
                 Id.serializer.serialize(request.readId, out, version);
                 ReadCommand.serializer.serialize(request.command, out, version);
+                out.writeInt(request.dataNode);
+                out.writeInt(request.summaryNodes.length);
+                for (int hostid : request.summaryNodes)
+                    out.writeInt(hostid);
             }
 
             @Override
@@ -512,14 +539,21 @@ public abstract class TrackedRead<E extends Endpoints<E>, P extends ReplicaPlan.
             {
                 Id readId = Id.serializer.deserialize(in, version);
                 ReadCommand command = ReadCommand.serializer.deserialize(in, version);
-                return new SummaryRequest(readId, command);
+                int dataNode = in.readInt();
+                int[] summaryNodes = new int[in.readInt()];
+                for (int i = 0; i < summaryNodes.length; i++)
+                    summaryNodes[i] = in.readInt();
+                return new SummaryRequest(readId, command, dataNode, summaryNodes);
             }
 
             @Override
             public long serializedSize(SummaryRequest request, int version)
             {
                 return Id.serializer.serializedSize(request.readId, version) +
-                       ReadCommand.serializer.serializedSize(request.command, version);
+                       ReadCommand.serializer.serializedSize(request.command, version) +
+                       TypeSizes.sizeof(request.dataNode) +
+                       TypeSizes.sizeof(request.summaryNodes.length) +
+                       ((long) TypeSizes.INT_SIZE * request.summaryNodes.length);
             }
         };
     }
diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
index 97f2254eee..ac6a4ddbec 100644
--- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
+++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java
@@ -17,38 +17,42 @@
  */
 package org.apache.cassandra.service.reads.tracked;
 
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.replication.MutationSummary;
-import org.apache.cassandra.replication.MutationTrackingService;
 
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class TrackedSummaryResponse
 {
-    private final TrackedRead.Id readId;
-    private final MutationSummary summary;
+    private static final Logger logger = LoggerFactory.getLogger(TrackedSummaryResponse.class);
+
+    public final TrackedRead.Id readId;
+    public final MutationSummary summary;
+    public final int dataNode;
+    public final int[] summaryNodes;
 
-    public TrackedSummaryResponse(TrackedRead.Id readId, MutationSummary summary)
+    public TrackedSummaryResponse(TrackedRead.Id readId, MutationSummary summary, int dataNode, int[] summaryNodes)
     {
         this.readId = readId;
         this.summary = summary;
+        this.dataNode = dataNode;
+        this.summaryNodes = summaryNodes;
     }
 
-    public TrackedRead.Id readId()
-    {
-        return readId;
-    }
-
-    public MutationSummary summary()
+    public static final IVerbHandler<TrackedSummaryResponse> verbHandler = message ->
     {
-        return summary;
-    }
-
-    public static final IVerbHandler<TrackedSummaryResponse> verbHandler =
-        message -> MutationTrackingService.instance.localReads().receiveSummary(message.from(), message.payload);
+        TrackedSummaryResponse response = message.payload;
+        if (logger.isTraceEnabled())
+            logger.trace("Received summary {} from {}, for {}", response.summary, message.from(), response.readId);
+        ReadReconciliations.instance.acceptRemoteSummary(message.from(), message.payload);
+    };
 
     public static final IVersionedSerializer<TrackedSummaryResponse> serializer = new IVersionedSerializer<>()
     {
@@ -57,6 +61,10 @@ public class TrackedSummaryResponse
         {
             TrackedRead.Id.serializer.serialize(summary.readId, out, version);
             MutationSummary.serializer.serialize(summary.summary, out, version);
+            out.writeInt(summary.dataNode);
+            out.writeInt(summary.summaryNodes.length);
+            for (int hostid : summary.summaryNodes)
+                out.writeInt(hostid);
         }
 
         @Override
@@ -64,14 +72,21 @@ public class TrackedSummaryResponse
         {
             TrackedRead.Id id = TrackedRead.Id.serializer.deserialize(in, version);
             MutationSummary summary = MutationSummary.serializer.deserialize(in, version);
-            return new TrackedSummaryResponse(id, summary);
+            int dataNode = in.readInt();
+            int[] summaryNodes = new int[in.readInt()];
+            for (int i = 0; i < summaryNodes.length; i++)
+                summaryNodes[i] = in.readInt();
+            return new TrackedSummaryResponse(id, summary, dataNode, summaryNodes);
         }
 
         @Override
         public long serializedSize(TrackedSummaryResponse summary, int version)
         {
             return TrackedRead.Id.serializer.serializedSize(summary.readId, version) +
-                   MutationSummary.serializer.serializedSize(summary.summary, version);
+                   MutationSummary.serializer.serializedSize(summary.summary, version) +
+                   TypeSizes.sizeof(summary.dataNode) +
+                   TypeSizes.sizeof(summary.summaryNodes.length) +
+                   TypeSizes.INT_SIZE * (long) summary.summaryNodes.length;
         }
     };
 }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
index 009399beb5..aa93dd9fec 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java
@@ -77,9 +77,9 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
 import static org.apache.cassandra.distributed.shared.AssertUtils.assertEquals;
 import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
 import static org.apache.cassandra.distributed.shared.AssertUtils.row;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_NOTIFY;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_RCV;
-import static org.apache.cassandra.net.Verb.READ_RECONCILE_SEND;
+import static org.apache.cassandra.net.Verb.PULL_MUTATIONS_REQ;
+import static org.apache.cassandra.net.Verb.PUSH_MUTATION_REQ;
+import static org.apache.cassandra.net.Verb.READ_RECONCILE_ACK;
 import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
 import static org.apache.cassandra.net.Verb.READ_REPAIR_RSP;
 import static org.apache.cassandra.net.Verb.READ_REQ;
@@ -128,6 +128,7 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
     @Test
     public void testBlockingReadRepairAccord() throws Throwable
     {
+        MutationTrackingUtils.fixmeSkipIfTracked(replicationType(), "Accord not supported");
         testReadRepair(ReadRepairStrategy.BLOCKING, true);
     }
 
@@ -182,7 +183,7 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
             cluster.get(1).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)"));
             cluster.get(2).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)"));
             assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1")));
-            cluster.verbs(replicationType().isTracked() ? READ_RECONCILE_NOTIFY : READ_REPAIR_RSP).to(1).drop();
+            cluster.verbs(replicationType().isTracked() ? READ_RECONCILE_ACK : READ_REPAIR_RSP).to(1).drop();
             final long start = currentTimeMillis();
             try
             {
@@ -219,9 +220,10 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
             assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1")));
 
             cluster.filters().verbs(READ_REPAIR_REQ.id).to(3).drop();
-            cluster.filters().verbs(READ_RECONCILE_SEND.id).to(3).drop();
-            cluster.filters().verbs(READ_RECONCILE_RCV.id).to(3).drop();
-            assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"), ConsistencyLevel.QUORUM),
+            cluster.filters().verbs(PULL_MUTATIONS_REQ.id).to(3).drop();
+            cluster.filters().verbs(PUSH_MUTATION_REQ.id).to(3).drop();
+            assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"),
+                                                      ConsistencyLevel.QUORUM),
                        row(1, 1, 1));
 
             // Data was not repaired
@@ -268,8 +270,8 @@ public abstract class ReadRepairTestBase extends TestBaseImpl
             cluster.filters().verbs(READ_REQ.id).from(4).to(3).drop();
             if (replicationType().isTracked())
             {
-                cluster.filters().verbs(READ_RECONCILE_SEND.id).from(4).to(3).drop();
-                cluster.filters().verbs(READ_RECONCILE_RCV.id).from(4).to(3).drop();
+                cluster.filters().verbs(PULL_MUTATIONS_REQ.id).from(4).to(3).drop();
+                cluster.filters().verbs(PUSH_MUTATION_REQ.id).from(4).to(3).drop();
             }
             else
             {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
index 9b420b7d22..e599b982a1 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTester.java
@@ -146,8 +146,13 @@ public abstract class ReadRepairTester<T extends ReadRepairTester<T>>
         // verify the number of repaired rows
         if (strategy == ReadRepairStrategy.NONE)
             expectedRepaired = 0;
-        assertEquals(String.format("Expected %d repaired rows, but found %d", expectedRepaired, actualRepaired),
-                     expectedRepaired, actualRepaired);
+
+        // read repair metric isn't a meaningful one for mutation-tracked replication
+        if (!replicationType.isTracked())
+        {
+            assertEquals(String.format("Expected %d repaired rows, but found %d", expectedRepaired, actualRepaired),
+                         expectedRepaired, actualRepaired);
+        }
 
         return self();
     }
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
index 7d859ce110..ae49c00383 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java
@@ -45,7 +45,7 @@ import org.apache.cassandra.replication.MutationTrackingService;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.reads.tracked.PartialTrackedRead;
-import org.apache.cassandra.service.reads.tracked.TrackedLocalReadCoordinator;
+import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static java.lang.String.format;
@@ -150,7 +150,7 @@ public class MutationTrackingPendingReadTest
                     // Create another summary once initial data has been read fully. We do this to catch
                     // any mutations that may have arrived during initial read execution.
                     secondarySummary = command.createMutationSummary(true);
-                    TrackedLocalReadCoordinator.processDelta(read, initialSummary, secondarySummary);
+                    TrackedLocalReads.processDelta(read, initialSummary, secondarySummary);
                 }
 
                 ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
index 21d18c5b08..8e6a8e108f 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.distributed.test.tracking;
 
 import java.util.UUID;
 
+import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.replication.CoordinatorLogId;
 import org.apache.cassandra.replication.MutationSummary;
 import org.apache.cassandra.replication.Offsets;
@@ -97,8 +98,6 @@ public class MutationTrackingTest extends TestBaseImpl
                                                             .set("write_request_timeout", "1000ms"))
                                       .start())
         {
-
-            String keyspaceName = KEYSPACE;
             cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
                                               "{'class': 'SimpleStrategy', 'replication_factor': 3} " +
                                               "AND replication_type='tracked';"));
@@ -124,4 +123,59 @@ public class MutationTrackingTest extends TestBaseImpl
             });
         }
     }
+
+    @Test
+    public void testFailedMutationRedelivery() throws Throwable
+    {
+        try (Cluster cluster = Cluster.build(3)
+                                      .withConfig(cfg -> cfg.with(Feature.NETWORK)
+                                                            .with(Feature.GOSSIP)
+                                                            .set("mutation_tracking_enabled", "true")
+                                                            .set("write_request_timeout", "1000ms"))
+                                      .start())
+        {
+            String keyspaceName = KEYSPACE;
+
+            cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
+                                              "{'class': 'SimpleStrategy', 'replication_factor': 3} " +
+                                              "AND replication_type='tracked';"));
+
+            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
+
+            // block writes to node 3
+            cluster.filters().verbs(Verb.MUTATION_REQ.id).to(3).drop();
+
+            // pause reconciler temporarily
+            cluster.get(1).runOnInstance(() -> MutationTrackingService.instance.pauseActiveReconciler());
+
+            // issue a write - should fail on node 3
+            cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM);
+
+            Thread.sleep(1000); // wait for write timeout
+
+            cluster.get(1).runOnInstance(() ->
+            {
+                TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
+                DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
+                MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
+                CoordinatorLogId logId = getOnlyLogId(summary);
+                Assert.assertEquals(1, summary.get(logId).unreconciled.offsetCount());
+                Assert.assertEquals(0, summary.get(logId).reconciled.offsetCount());
+            });
+
+            // resume the reconciler
+            cluster.get(1).runOnInstance(() -> MutationTrackingService.instance.resumeActiveReconciler());
+            Thread.sleep(1000); // wait for reconiciler to do its job
+
+            cluster.get(1).runOnInstance(() ->
+            {
+                TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
+                DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
+                MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
+                CoordinatorLogId logId = getOnlyLogId(summary);
+                Assert.assertEquals(0, summary.get(logId).unreconciled.offsetCount());
+                Assert.assertEquals(1, summary.get(logId).reconciled.offsetCount());
+            });
+        }
+    }
 }
diff --git a/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java b/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
index 1d25a1956f..91a2788ddf 100644
--- a/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
+++ b/test/unit/org/apache/cassandra/replication/CoordinatorLogOffsetsTest.java
@@ -274,8 +274,8 @@ public class CoordinatorLogOffsetsTest
             MutationTrackingService.instance.startWriting(mutation);
 
             MutationTrackingService.instance.finishWriting(mutation);
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
 
             ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
                     .add(mutation.id())
@@ -300,8 +300,8 @@ public class CoordinatorLogOffsetsTest
             Mutation mutation = MutationTrackingUtils.createMutation(tableMetadata, 3, 3);
             MutationTrackingService.instance.startWriting(mutation);
 
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
 
             ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
                     .add(mutation.id())
@@ -315,8 +315,8 @@ public class CoordinatorLogOffsetsTest
             MutationTrackingService.instance.startWriting(mutation);
 
             MutationTrackingService.instance.finishWriting(mutation);
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr2);
-            MutationTrackingService.instance.receivedWriteResponse(ks, mutation.key().getToken(), mutation.id(), addr3);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr2);
+            MutationTrackingService.instance.receivedWriteResponse(mutation.id(), addr3);
 
             MutationId fakeMutationId = new MutationId(CoordinatorLogId.asLong(111, 222), MutationId.sequenceId(333, 444));
             Assertions.assertThat(metadata.directory.version(new NodeId(fakeMutationId.hostId()))).isNull();
diff --git a/test/unit/org/apache/cassandra/replication/OffsetsTest.java b/test/unit/org/apache/cassandra/replication/OffsetsTest.java
index d7c4d4fcb1..5d10ae5e45 100644
--- a/test/unit/org/apache/cassandra/replication/OffsetsTest.java
+++ b/test/unit/org/apache/cassandra/replication/OffsetsTest.java
@@ -105,7 +105,7 @@ public class OffsetsTest
 
         public TestConsumer assertOffsetsConsumed(int... expected)
         {
-            Assert.assertTrue(expected.length % 2 == 0);
+            assertEquals(0, expected.length % 2);
             TestConsumer expectedConsumer = new TestConsumer();
             for (int i = 0; i < expected.length; i+=2)
                 expectedConsumer.consumerOffsets(expected[i], expected[i+1]);
@@ -134,7 +134,7 @@ public class OffsetsTest
 
     private static Offsets.Mutable offsets(int... bounds)
     {
-        Assert.assertTrue(bounds.length % 2 == 0);
+        assertEquals(0, bounds.length % 2);
         Offsets.Mutable ids = new Offsets.Mutable(LOG_ID);
         int keys = 0;
         int last = 0;
@@ -506,6 +506,41 @@ public class OffsetsTest
         }
     }
 
+    @Test
+    public void testMultiMerge()
+    {
+        Supplier<Offsets.Mutable> sequenceIds = () -> {
+            Offsets.Mutable ids0 = new Offsets.Mutable(LOG_ID);
+            ids0.add(0, 3);
+            ids0.add(7, 10);
+            ids0.add(15, 17);
+
+            assertEquals(3, ids0.rangeCount());
+            assertEquals(11, ids0.offsetCount());
+            return ids0;
+        };
+
+        {
+            // extends on the end
+            Offsets.Mutable offsets = sequenceIds.get();
+            TestConsumer consumer = new TestConsumer();
+
+            assertTrue(offsets.add(0, 12, consumer));
+            consumer.assertOffsetsConsumed(4, 6, 11, 12).clear();
+            assertEquals(offsets(0, 12, 15, 17), offsets);
+        }
+
+        {
+            // extends on the start
+            Offsets.Mutable offsets = sequenceIds.get();
+            TestConsumer consumer = new TestConsumer();
+
+            assertTrue(offsets.add(5, 17, consumer));
+            consumer.assertOffsetsConsumed(5, 6, 11, 14).clear();
+            assertEquals(offsets(0, 3, 5, 17), offsets);
+        }
+    }
+
     @Test
     public void addTest()
     {
@@ -747,4 +782,60 @@ public class OffsetsTest
         Assert.assertEquals(ids(1, 2, 3), ids(offsets(1, 3)));
         Assert.assertEquals(ids(1, 2, 3, 5, 6, 7), ids(offsets(1, 3, 5, 7)));
     }
+
+    @Test
+    public void testRemoveFromEmpty()
+    {
+        testRemove(offsets(), 5, offsets());
+    }
+
+    @Test
+    public void testRemoveFromSingleRange()
+    {
+        // before all ranges
+        testRemove(offsets(5, 7), 4, offsets(5, 7));
+        // after all ranges
+        testRemove(offsets(5, 7), 8, offsets(5, 7));
+        // start of range
+        testRemove(offsets(5, 7), 5, offsets(6, 7));
+        // end of range
+        testRemove(offsets(5, 7), 7, offsets(5, 6));
+        // middle of range
+        testRemove(offsets(5, 7), 6, offsets(5, 5, 7, 7));
+        // single element range
+        testRemove(offsets(5, 5), 5, offsets());
+    }
+
+    @Test
+    public void testRemoveGeneric()
+    {
+        // before all ranges
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 4, offsets(5, 7, 9, 9, 11, 13));
+        // after all ranges
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 14, offsets(5, 7, 9, 9, 11, 13));
+        // between two ranges
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 8, offsets(5, 7, 9, 9, 11, 13));
+        // start of a range
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 11, offsets(5, 7, 9, 9, 12, 13));
+        // end of a range
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 7, offsets(5, 6, 9, 9, 11, 13));
+        // middle of a range
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 12, offsets(5, 7, 9, 9, 11, 11, 13, 13));
+        // single element range
+        testRemove(offsets(5, 7, 9, 9, 11, 13), 9, offsets(5, 7, 11, 13));
+
+        // single element ranges at different positions
+        testRemove(offsets(5, 5, 10, 12), 5, offsets(10, 12));
+        testRemove(offsets(1, 3, 5, 5, 7, 9), 5, offsets(1, 3, 7, 9));
+
+        // range splitting at boundaries
+        testRemove(offsets(5, 10), 7, offsets(5, 6, 8, 10));
+        testRemove(offsets(5, 10, 15, 17), 7, offsets(5, 6, 8, 10, 15, 17));
+    }
+
+    private void testRemove(Offsets.Mutable from, int toRemove, Offsets.Mutable expectedAfter)
+    {
+        from.remove(toRemove);
+        assertOffsetsEqual(expectedAfter, from);
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org