You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2024/09/30 15:01:34 UTC

(cassandra-accord) 01/09: squash 19869 to 80414c2e

This is an automated email from the ASF dual-hosted git repository.

ifesdjeen pushed a commit to branch ex-sp-pl
in repository https://gitbox.apache.org/repos/asf/cassandra-accord.git

commit 34c35d8c7678f0aa4c8f57c473fc8871851534a5
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Tue Jul 30 20:57:04 2024 +0200

    squash 19869 to 80414c2e
---
 accord-core/build.gradle                           |   8 +-
 .../src/main/java/accord/api/DataStore.java        |   2 +
 .../src/main/java/accord/api/ProgressLog.java      |   3 +
 .../main/java/accord/coordinate/ExecuteTxn.java    |   9 +
 .../coordinate/tracking/AbstractTracker.java       |   2 +-
 .../java/accord/impl/InMemoryCommandStore.java     | 256 +++++++---
 .../main/java/accord/impl/TimestampsForKeys.java   |   2 +-
 .../accord/impl/progresslog/CallbackInvoker.java   |   6 +
 .../impl/progresslog/DefaultProgressLog.java       |  23 +-
 .../java/accord/impl/progresslog/WaitingState.java |   3 +-
 .../src/main/java/accord/local/Bootstrap.java      |  34 +-
 .../src/main/java/accord/local/Cleanup.java        |  14 +-
 .../src/main/java/accord/local/Command.java        |   5 +-
 .../src/main/java/accord/local/CommandStore.java   | 248 ++++++----
 .../src/main/java/accord/local/CommandStores.java  |  50 +-
 .../src/main/java/accord/local/Commands.java       |  67 ++-
 .../src/main/java/accord/local/DurableBefore.java  |   5 +
 .../src/main/java/accord/local/PreLoadContext.java |  61 ++-
 .../main/java/accord/local/RedundantBefore.java    |  41 +-
 .../main/java/accord/local/SafeCommandStore.java   |  68 ++-
 .../main/java/accord/local/cfk/CommandsForKey.java |  18 +-
 .../src/main/java/accord/local/cfk/NotifySink.java |   5 +-
 .../src/main/java/accord/local/cfk/Updating.java   |   5 +-
 .../src/main/java/accord/messages/PreAccept.java   |   1 +
 .../java/accord/messages/SetGloballyDurable.java   |   5 +-
 .../src/main/java/accord/primitives/Writes.java    |   4 +
 .../src/main/java/accord/topology/Topologies.java  |  42 +-
 .../src/main/java/accord/topology/Topology.java    |  10 +-
 .../src/main/java/accord/utils/LogGroupTimers.java |  21 +
 .../java/accord/utils/TriConsumer.java}            |  14 +-
 .../main/java/accord/utils/async/AsyncChain.java   |  11 +
 .../test/java/accord/impl/RemoteListenersTest.java |  13 +-
 .../src/test/java/accord/impl/basic/Cluster.java   | 135 +++++-
 .../accord/impl/basic/DelayedCommandStores.java    |  32 +-
 .../src/test/java/accord/impl/basic/Journal.java   | 518 ++++++++++++---------
 .../src/test/java/accord/impl/basic/NodeSink.java  |   9 +-
 .../test/java/accord/impl/basic/PendingQueue.java  |   7 +-
 .../accord/impl/basic/PropagatingPendingQueue.java |  20 +
 .../java/accord/impl/basic/RandomDelayQueue.java   |  54 ++-
 .../impl/basic/RecurringPendingRunnable.java       |  14 +-
 .../accord/impl/list/ListFetchCoordinator.java     |   2 +-
 .../src/test/java/accord/impl/list/ListStore.java  | 104 ++++-
 .../src/test/java/accord/impl/mock/MockStore.java  |   8 +
 .../java/accord/local/BootstrapLocalTxnTest.java   | 127 -----
 .../java/accord/local/cfk/CommandsForKeyTest.java  |  28 +-
 .../verify/StrictSerializabilityVerifier.java      |   4 +-
 .../main/java/accord/maelstrom/MaelstromStore.java |   9 +
 47 files changed, 1416 insertions(+), 711 deletions(-)

diff --git a/accord-core/build.gradle b/accord-core/build.gradle
index d87e5000..b30737c1 100644
--- a/accord-core/build.gradle
+++ b/accord-core/build.gradle
@@ -58,7 +58,7 @@ dependencies {
 task burn(type: JavaExec) {
     classpath sourceSets.main.runtimeClasspath, sourceSets.test.runtimeClasspath
     mainClass = 'accord.burn.BurnTest'
-    jvmArgs '-Dlogback.configurationFile=burn-logback.xml'
+    jvmArgs '-Dlogback.configurationFile=burn-logback.xml -Xmx4g'
     args = ['-c', '1']
 }
 
@@ -122,7 +122,7 @@ task burnloop {
         javaexec {
             classpath sourceSets.main.runtimeClasspath, sourceSets.test.runtimeClasspath
             mainClass = 'accord.burn.BurnTest'
-            jvmArgs '-Dlogback.configurationFile=burn-logback.xml'
+            jvmArgs '-Dlogback.configurationFile=burn-logback.xml -Xmx4g'
             args = ['-c', project.hasProperty('burnTimes') ? project.getProperty('burnTimes') : '1000000']
         }
     }
@@ -142,4 +142,8 @@ task allTestJar(type: Jar) {
     from { configurations.testRuntimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
     from { sourceSets.test.output }
     with jar
+}
+
+test {
+    maxHeapSize = '4g'
 }
\ No newline at end of file
diff --git a/accord-core/src/main/java/accord/api/DataStore.java b/accord-core/src/main/java/accord/api/DataStore.java
index d71c742b..623b021f 100644
--- a/accord-core/src/main/java/accord/api/DataStore.java
+++ b/accord-core/src/main/java/accord/api/DataStore.java
@@ -25,6 +25,7 @@ import accord.local.SafeCommandStore;
 import accord.primitives.Ranges;
 import accord.primitives.SyncPoint;
 import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
 import accord.utils.async.AsyncResult;
 
 /**
@@ -111,4 +112,5 @@ public interface DataStore
     }
 
     FetchResult fetch(Node node, SafeCommandStore safeStore, Ranges ranges, SyncPoint syncPoint, FetchRanges callback);
+    AsyncResult<Void> snapshot(Ranges ranges, TxnId before);
 }
diff --git a/accord-core/src/main/java/accord/api/ProgressLog.java b/accord-core/src/main/java/accord/api/ProgressLog.java
index 3a82ccd3..6bdcefae 100644
--- a/accord-core/src/main/java/accord/api/ProgressLog.java
+++ b/accord-core/src/main/java/accord/api/ProgressLog.java
@@ -185,11 +185,14 @@ public interface ProgressLog
      */
     void clear(TxnId txnId);
 
+    void clear();
+
     class NoOpProgressLog implements ProgressLog
     {
         @Override public void update(SafeCommandStore safeStore, TxnId txnId, Command before, Command after) {}
         @Override public void remoteCallback(SafeCommandStore safeStore, SafeCommand safeCommand, SaveStatus remoteStatus, int callbackId, Node.Id from) {}
         @Override public void waiting(BlockedUntil blockedUntil, SafeCommandStore safeStore, SafeCommand blockedBy, Route<?> blockedOnRoute, Participants<?> blockedOnParticipants) {}
         @Override public void clear(TxnId txnId) {}
+        @Override public void clear() {}
     }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
index 34d3e095..3f689156 100644
--- a/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
+++ b/accord-core/src/main/java/accord/coordinate/ExecuteTxn.java
@@ -159,4 +159,13 @@ public class ExecuteTxn extends ReadCoordinator<ReadReply>
     {
         return node.coordinationAdapter(txnId, Continue);
     }
+
+    @Override
+    public String toString()
+    {
+        return "ExecuteTxn{" +
+               "txn=" + txn +
+               ", route=" + route +
+               '}';
+    }
 }
diff --git a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
index e58f3c67..183b4879 100644
--- a/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
+++ b/accord-core/src/main/java/accord/coordinate/tracking/AbstractTracker.java
@@ -180,7 +180,7 @@ public abstract class AbstractTracker<ST extends ShardTracker>
         return topologies.nodes();
     }
 
-    public Set<Id> nonStaleNodes()
+    public Collection<Id> nonStaleNodes()
     {
         return topologies.nonStaleNodes();
     }
diff --git a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
index bcc6d6f7..d932f6b8 100644
--- a/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
+++ b/accord-core/src/main/java/accord/impl/InMemoryCommandStore.java
@@ -42,22 +42,33 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-
-import accord.api.LocalListeners;
-import accord.impl.progresslog.DefaultProgressLog;
-import accord.local.*;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.Key;
+import accord.api.LocalListeners;
 import accord.api.ProgressLog;
+import accord.impl.progresslog.DefaultProgressLog;
+import accord.local.Cleanup;
+import accord.local.Command;
+import accord.local.CommandStore;
 import accord.local.CommandStores.RangesForEpoch;
+import accord.local.Commands;
+import accord.local.KeyHistory;
+import accord.local.Node;
+import accord.local.NodeTimeService;
+import accord.local.PreLoadContext;
+import accord.local.RedundantStatus;
+import accord.local.SafeCommand;
+import accord.local.SafeCommandStore;
+import accord.local.SaveStatus;
+import accord.local.Status;
 import accord.local.cfk.CommandsForKey;
 import accord.primitives.AbstractKeys;
 import accord.primitives.Deps;
+import accord.primitives.Keys;
 import accord.primitives.PartialDeps;
 import accord.primitives.Participants;
 import accord.primitives.Range;
@@ -73,22 +84,24 @@ import accord.primitives.Timestamp;
 import accord.primitives.Txn.Kind.Kinds;
 import accord.primitives.TxnId;
 import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
 
 import static accord.local.SafeCommandStore.TestDep.ANY_DEPS;
 import static accord.local.SafeCommandStore.TestDep.WITH;
-import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
 import static accord.local.SafeCommandStore.TestStartedAt.STARTED_BEFORE;
+import static accord.local.SafeCommandStore.TestStatus.ANY_STATUS;
 import static accord.local.SaveStatus.Applying;
 import static accord.local.SaveStatus.Erased;
 import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
 import static accord.local.SaveStatus.ReadyToExecute;
 import static accord.local.Status.Applied;
+import static accord.local.Status.Invalidated;
+import static accord.local.Status.NotDefined;
 import static accord.local.Status.PreCommitted;
 import static accord.local.Status.Stable;
 import static accord.local.Status.Truncated;
-import static accord.local.Status.NotDefined;
 import static accord.primitives.Routables.Slice.Minimal;
 import static accord.utils.Invariants.illegalState;
 import static java.lang.String.format;
@@ -102,10 +115,11 @@ public abstract class InMemoryCommandStore extends CommandStore
     final NavigableMap<Timestamp, GlobalCommand> commandsByExecuteAt = new TreeMap<>();
     private final NavigableMap<RoutableKey, GlobalTimestampsForKey> timestampsForKey = new TreeMap<>();
     private final NavigableMap<RoutableKey, GlobalCommandsForKey> commandsForKey = new TreeMap<>();
-    // TODO (find library, efficiency): this is obviously super inefficient, need some range map
 
+    // TODO (find library, efficiency): this is obviously super inefficient, need some range map
     private final TreeMap<TxnId, RangeCommand> rangeCommands = new TreeMap<>();
     private final TreeMap<TxnId, Ranges> historicalRangeCommands = new TreeMap<>();
+    // TODO (desired): use `redundantBefore` information instead
     protected Timestamp maxRedundant = Timestamp.NONE;
 
     private InMemorySafeStore current;
@@ -149,11 +163,6 @@ public abstract class InMemoryCommandStore extends CommandStore
         return historicalRangeCommands;
     }
 
-    TreeMap<TxnId, RangeCommand> rangeCommands()
-    {
-        return rangeCommands;
-    }
-
     public GlobalCommand commandIfPresent(TxnId txnId)
     {
         return commands.get(txnId);
@@ -362,7 +371,11 @@ public abstract class InMemoryCommandStore extends CommandStore
     public void markShardDurable(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
     {
         super.markShardDurable(safeStore, syncId, ranges);
+        markShardDurable(syncId, ranges);
+    }
 
+    private void markShardDurable(TxnId syncId, Ranges ranges)
+    {
         if (!rangeCommands.containsKey(syncId))
             historicalRangeCommands.merge(syncId, ranges, Ranges::with);
 
@@ -408,8 +421,6 @@ public abstract class InMemoryCommandStore extends CommandStore
         }, 5L, TimeUnit.SECONDS);
     }
 
-
-
     protected InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges,
                                                 Map<TxnId, InMemorySafeCommand> commands,
                                                 Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey,
@@ -418,15 +429,6 @@ public abstract class InMemoryCommandStore extends CommandStore
         return new InMemorySafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys);
     }
 
-    private void loadCommandsForKey(RoutableKey key,
-                                    KeyHistory keyHistory,
-                                    Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey,
-                                    Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey)
-    {
-        commandsForKey.put(key, commandsForKey((Key) key).createSafeReference());
-        timestampsForKey.put(key, timestampsForKey((Key) key).createSafeReference());
-    }
-
     protected void validateRead(Command current) {}
 
     protected final InMemorySafeStore createSafeStore(PreLoadContext context, RangesForEpoch ranges)
@@ -475,7 +477,8 @@ public abstract class InMemoryCommandStore extends CommandStore
     {
         if (current != null)
             throw illegalState("Another operation is in progress or it's store was not cleared");
-        current = createSafeStore(context, updateRangesForEpoch());
+        current = createSafeStore(context, rangesForEpoch);
+        updateRangesForEpoch(current);
         return current;
     }
 
@@ -509,7 +512,7 @@ public abstract class InMemoryCommandStore extends CommandStore
         catch (Throwable t)
         {
             if (isDirectCall) logger.error("Uncaught exception", t);
-            throw t;
+            throw new RuntimeException("Caught exception in command store " + this, t);
         }
         finally
         {
@@ -658,7 +661,7 @@ public abstract class InMemoryCommandStore extends CommandStore
         protected final Map<TxnId, InMemorySafeCommand> commands;
         private final Map<RoutableKey, InMemorySafeTimestampsForKey> timestampsForKey;
         private final Map<RoutableKey, InMemorySafeCommandsForKey> commandsForKey;
-        private final RangesForEpoch ranges;
+        private RangesForEpoch ranges;
 
         public InMemorySafeStore(InMemoryCommandStore commandStore,
                                  RangesForEpoch ranges,
@@ -672,7 +675,7 @@ public abstract class InMemoryCommandStore extends CommandStore
             this.commands = commands;
             this.commandsForKey = commandsForKey;
             this.timestampsForKey = timestampsForKey;
-            this.ranges = Invariants.nonNull(ranges);
+            this.ranges = ranges;
         }
 
         @Override
@@ -734,7 +737,7 @@ public abstract class InMemoryCommandStore extends CommandStore
         {
             if (!commandStore.canExposeUnloaded())
                 return null;
-            GlobalCommandsForKey global = commandStore.commandsForKeyIfPresent((Key) key);
+            GlobalCommandsForKey global = commandStore.commandsForKeyIfPresent(key);
             return global != null ? global.createSafeReference() : null;
         }
 
@@ -760,7 +763,6 @@ public abstract class InMemoryCommandStore extends CommandStore
             slice = commandStore.redundantBefore().removeShardRedundant(txnId, updated.executeAtOrTxnId(), slice);
             commandStore.rangeCommands.computeIfAbsent(txnId, ignore -> new RangeCommand(commandStore.commands.get(txnId)))
                          .update(((Ranges)keysOrRanges).slice(slice, Minimal));
-
         }
 
         @Override
@@ -793,41 +795,11 @@ public abstract class InMemoryCommandStore extends CommandStore
             return ranges;
         }
 
-        // TODO (desired): this can have protected visibility if under CommandStore, and this is perhaps a better place to put it also
         @Override
-        public void registerHistoricalTransactions(Deps deps)
-        {
-            RangesForEpoch rangesForEpoch = commandStore.rangesForEpoch;
-            Ranges allRanges = rangesForEpoch.all();
-            deps.keyDeps.keys().forEach(allRanges, key -> {
-                deps.keyDeps.forEach(key, (txnId, txnIdx) -> {
-                    // TODO (desired, efficiency): this can be made more efficient by batching by epoch
-                    if (rangesForEpoch.coordinates(txnId).contains(key))
-                        return; // already coordinates, no need to replicate
-                    // TODO (required): check this logic, esp. next line, matches C*
-                    if (!rangesForEpoch.allAfter(txnId.epoch()).contains(key))
-                        return;
-
-                    get(key).registerHistorical(this, txnId);
-                });
-
-            });
-            TreeMap<TxnId, RangeCommand> rangeCommands = commandStore.rangeCommands();
-            TreeMap<TxnId, Ranges> historicalRangeCommands = commandStore.historicalRangeCommands();
-            deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
-
-                if (rangeCommands.containsKey(txnId))
-                    return;
-
-                Ranges ranges = deps.rangeDeps.ranges(txnId);
-                if (rangesForEpoch.coordinates(txnId).intersects(ranges))
-                    return; // already coordinates, no need to replicate
-                // TODO (required): check this logic, esp. next line, matches C*
-                if (!rangesForEpoch.allAfter(txnId.epoch()).intersects(ranges))
-                    return;
-
-                historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
-            });
+        public void setRangesForEpoch(RangesForEpoch rangesForEpoch)
+        {
+            super.setRangesForEpoch(rangesForEpoch);
+            ranges = rangesForEpoch;
         }
 
         @Override
@@ -1338,4 +1310,160 @@ public abstract class InMemoryCommandStore extends CommandStore
             logDependencyGraph(commandStore, txnId, true);
         }
     }
+
+    /**
+     * Replay and loading logic
+     */
+
+    // redundantBefore, durableBefore, newBootstrapBeganAt, safeToRead, rangesForEpoch are
+    // not replayed here. It is assumed that persistence on the application side will ensure
+    // they are brought up to latest values _before_ replay.
+    public void clearForTesting()
+    {
+        Invariants.checkState(current == null);
+        progressLog.clear();
+        commands.clear();
+        commandsByExecuteAt.clear();
+        timestampsForKey.clear();
+        commandsForKey.clear();
+        rangeCommands.clear();
+        historicalRangeCommands.clear();
+        unsafeSetRejectBefore(new ReducingRangeMap<>());
+    }
+
+    public interface Loader
+    {
+        void load(Command next);
+        void apply(Command next);
+    }
+
+    public Loader loader()
+    {
+        return new Loader()
+        {
+            private PreLoadContext context(Command command, KeyHistory keyHistory)
+            {
+                TxnId txnId = command.txnId();
+                Keys keys = null;
+
+                if (CommandsForKey.manages(txnId))
+                    keys = (Keys) command.keysOrRanges();
+                else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
+                    keys = command.asCommitted().waitingOn.keys;
+
+                if (keys != null)
+                {
+                    return PreLoadContext.contextFor(txnId, keys, keyHistory);
+                }
+
+                return PreLoadContext.contextFor(txnId);
+            }
+
+            public void load(Command command)
+            {
+                TxnId txnId = command.txnId();
+
+                executeInContext(InMemoryCommandStore.this,
+                                 context(command, KeyHistory.COMMANDS),
+                                 safeStore -> {
+                                     Command local = command;
+                                     if (local.status() != Truncated && local.status() != Invalidated)
+                                     {
+                                         Cleanup cleanup = Cleanup.shouldCleanup(InMemoryCommandStore.this, local, null, local.route(), false);
+                                         switch (cleanup)
+                                         {
+                                             case NO:
+                                                 break;
+                                             case INVALIDATE:
+                                             case TRUNCATE_WITH_OUTCOME:
+                                             case TRUNCATE:
+                                             case ERASE:
+                                                 local = Commands.purge(local, local.route(), cleanup);
+                                         }
+                                     }
+
+                                     local = safeStore.unsafeGet(txnId).update(safeStore, local);
+                                     if (local.status() == Truncated)
+                                         safeStore.progressLog().clear(local.txnId());
+                                     return local;
+                                 });
+
+
+            }
+
+            public void apply(Command command)
+            {
+                TxnId txnId = command.txnId();
+
+                PreLoadContext context = context(command, KeyHistory.TIMESTAMPS);
+                executeInContext(InMemoryCommandStore.this,
+                                 context,
+                                 safeStore -> {
+                                     SafeCommand safeCommand = safeStore.unsafeGet(txnId);
+                                     Command local = safeCommand.current();
+                                     if (local.is(Stable) && !local.hasBeen(Applied))
+                                         Commands.maybeExecute(safeStore, safeCommand, local, true, true);
+                                     else if (local.saveStatus().compareTo(Applying) >= 0 && !local.is(Invalidated) && !local.is(Truncated))
+                                         Commands.applyWrites(safeStore, context, local).begin(agent);
+                                     return null;
+                                 });
+            }
+        };
+    }
+
+    @VisibleForTesting
+    public void load(Deps loading)
+    {
+        registerHistoricalTransactions(loading,
+                                       ((key, txnId) -> {
+                                           GlobalCommandsForKey globalCfk = commandsForKey(key);
+                                           if (globalCfk.isEmpty())
+                                               globalCfk.value(new CommandsForKey(key));
+                                           CommandsForKey cfk = globalCfk.createSafeReference().current();
+                                           Invariants.checkState(cfk != null);
+                                           globalCfk.value(cfk.registerHistorical(txnId).cfk());
+                                       }));
+    }
+
+    @Override
+    protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+    {
+        registerHistoricalTransactions(deps, (key, txnId) -> safeStore.get(key).registerHistorical(safeStore, txnId));
+    }
+
+    private void registerHistoricalTransactions(Deps deps, BiConsumer<Key, TxnId> registerHistorical)
+    {
+        RangesForEpoch rangesForEpoch = this.rangesForEpoch;
+        Ranges allRanges = rangesForEpoch.all();
+        deps.keyDeps.keys().forEach(allRanges, key -> {
+            deps.keyDeps.forEach(key, (txnId, txnIdx) -> {
+                // TODO (desired, efficiency): this can be made more efficient by batching by epoch
+                if (rangesForEpoch.coordinates(txnId).contains(key))
+                    return; // already coordinates, no need to replicate
+                // TODO (required): check this logic, esp. next line, matches C*
+                if (!rangesForEpoch.allAfter(txnId.epoch()).contains(key))
+                    return;
+
+                registerHistorical.accept(key, txnId);
+            });
+
+        });
+
+        TreeMap<TxnId, RangeCommand> rangeCommands = this.rangeCommands;
+        TreeMap<TxnId, Ranges> historicalRangeCommands = historicalRangeCommands();
+        deps.rangeDeps.forEachUniqueTxnId(allRanges, null, (ignore, txnId) -> {
+
+            if (rangeCommands.containsKey(txnId))
+                return;
+
+            Ranges ranges = deps.rangeDeps.ranges(txnId);
+            if (rangesForEpoch.coordinates(txnId).intersects(ranges))
+                return; // already coordinates, no need to replicate
+            // TODO (required): check this logic, esp. next line, matches C*
+            if (!rangesForEpoch.allAfter(txnId.epoch()).intersects(ranges))
+                return;
+
+            historicalRangeCommands.merge(txnId, ranges.slice(allRanges), Ranges::with);
+        });
+    }
 }
diff --git a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
index 7dcfa6d4..d46d3513 100644
--- a/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
+++ b/accord-core/src/main/java/accord/impl/TimestampsForKeys.java
@@ -43,7 +43,7 @@ public class TimestampsForKeys
         {
             if (!commandStore.safeToReadAt(executeAt).contains(tfk.key().toUnseekable()))
                 return current;
-            throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s", executeAt, lastWrite));
+            throw new IllegalArgumentException(String.format("%s is less than the most recent write timestamp %s for %s", executeAt, lastWrite, tfk.key()));
         }
 
         Timestamp lastExecuted = current.lastExecutedTimestamp();
diff --git a/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java b/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
index afd6aea1..0170cc11 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/CallbackInvoker.java
@@ -100,5 +100,11 @@ class CallbackInvoker<P, V> implements BiConsumer<V, Throwable>
     {
         return txnId.hashCode();
     }
+
+    @Override
+    public String toString()
+    {
+        return txnId + (isHome ? ":Home:" : ":Waiting:") + owner.commandStore;
+    }
 }
 
diff --git a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
index 752a8684..0932d92e 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/DefaultProgressLog.java
@@ -79,8 +79,8 @@ public class DefaultProgressLog implements ProgressLog, Runnable
     private Object[] stateMap = BTree.empty();
     private Object[] progressTokenMap = BTree.empty();
 
-    private final LogGroupTimers<TxnState> timers = new LogGroupTimers<>(MICROSECONDS);
-    private final LogGroupTimers<TxnState>.Scheduling timerScheduling;
+    final LogGroupTimers<TxnState> timers = new LogGroupTimers<>(MICROSECONDS);
+    final LogGroupTimers<TxnState>.Scheduling timerScheduling;
 
     /**
      * A collection of active callbacks (waiting remote replies) or submitted run invocations
@@ -278,6 +278,23 @@ public class DefaultProgressLog implements ProgressLog, Runnable
             clear(state);
     }
 
+    public void clear()
+    {
+        timers.clear();
+        timerScheduling.clear();
+
+        stateMap = BTree.empty();
+        progressTokenMap = BTree.empty();
+
+        activeWaiting.clear();
+        activeHome.clear();
+        if (deleted != null)
+            deleted.clear();
+
+        runBuffer = EMPTY_RUN_BUFFER;
+        runBufferCount = 0;
+    }
+
     private void clear(TxnState state)
     {
         state.setHomeDone(this);
@@ -461,7 +478,7 @@ public class DefaultProgressLog implements ProgressLog, Runnable
             if (!deregisterActive(runKind, this))
                 return; // we've been cancelled
 
-            Invariants.checkState(get(run.txnId) == run);
+            Invariants.checkState(get(run.txnId) == run, "Transaction state for %s does not match expected one %s", run.txnId, run);
             Invariants.checkState(run.scheduledTimer() != runKind, "We are actively executing %s, but we are also scheduled to run this same TxnState later. This should not happen.", runKind);
             Invariants.checkState(run.pendingTimer() != runKind, "We are actively executing %s, but we also have a pending scheduled task to run this same TxnState later. This should not happen.", runKind);
 
diff --git a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
index 792eae23..d60f2a28 100644
--- a/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
+++ b/accord-core/src/main/java/accord/impl/progresslog/WaitingState.java
@@ -303,7 +303,8 @@ abstract class WaitingState extends BaseTxnState
         BlockedUntil blockedUntil = blockedUntil();
         Command command = safeCommand.current();
         Invariants.checkState(!owner.hasActive(Waiting, txnId));
-        Invariants.checkState(command.saveStatus().compareTo(blockedUntil.minSaveStatus) < 0, "Command has met desired criteria but progress log entry has not been cancelled");
+        Invariants.checkState(command.saveStatus().compareTo(blockedUntil.minSaveStatus) < 0,
+                              "Command has met desired criteria (%s) but progress log entry has not been cancelled: %s", blockedUntil.minSaveStatus, command);
 
         set(safeStore, owner, blockedUntil, Querying);
         TxnId txnId = safeCommand.txnId();
diff --git a/accord-core/src/main/java/accord/local/Bootstrap.java b/accord-core/src/main/java/accord/local/Bootstrap.java
index 08bf14a2..d77df6f5 100644
--- a/accord-core/src/main/java/accord/local/Bootstrap.java
+++ b/accord-core/src/main/java/accord/local/Bootstrap.java
@@ -41,7 +41,6 @@ import accord.utils.Invariants;
 import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 
-import static accord.local.PreLoadContext.contextFor;
 import static accord.local.PreLoadContext.empty;
 import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
@@ -96,7 +95,7 @@ class Bootstrap
     }
 
     // an attempt to fetch some portion of the range we are bootstrapping
-    class Attempt implements FetchRanges, BiConsumer<Void, Throwable>
+    class Attempt implements FetchRanges, BiConsumer<Object, Throwable>
     {
         final List<SafeToRead> states = new ArrayList<>();
         Runnable cancel;
@@ -117,7 +116,7 @@ class Bootstrap
             this.valid = ranges;
         }
 
-        void start(SafeCommandStore safeStore0)
+        void start(SafeCommandStore safeStore)
         {
             if (valid.isEmpty())
             {
@@ -142,23 +141,16 @@ class Bootstrap
             // we fix here the ranges we use for the synthetic command, even though we may end up only finishing a subset
             // of these ranges as part of this attempt
             Ranges commitRanges = valid;
-            store.markBootstrapping(safeStore0, globalSyncId, valid);
-            CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges)
-               // ATM all known implementations store ranges in-memory, but this will not be true soon, so this will need to be addressed
-               .flatMap(syncPoint -> node.withEpoch(epoch, () -> store.submit(contextFor(localSyncId, syncPoint.waitFor.keyDeps.keys(), KeyHistory.COMMANDS), safeStore1 -> {
-                   if (valid.isEmpty()) // we've lost ownership of the range
-                       return AsyncResults.success(Ranges.EMPTY);
-
-                   Commands.createBootstrapCompleteMarkerTransaction(safeStore1, localSyncId, valid);
-                   safeStore1.registerHistoricalTransactions(syncPoint.waitFor);
-                   return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
-               })))
-               .flatMap(i -> i)
-               .flatMap(ranges -> store.execute(contextFor(localSyncId), safeStore -> {
-                   if (!ranges.isEmpty())
-                       Commands.markBootstrapComplete(safeStore, localSyncId, ranges);
-               }))
-               .begin(this);
+            safeStore = safeStore;
+            // we submit a separate execution so that we know markBootstrapping is durable before we initiate the fetch
+            safeStore.commandStore().submit(empty(), safeStore0 -> {
+                store.markBootstrapping(safeStore0, globalSyncId, commitRanges);
+                return CoordinateSyncPoint.exclusive(node, globalSyncId, commitRanges);
+            }).flatMap(i -> i).flatMap(syncPoint -> node.withEpoch(epoch, () -> store.submit(empty(), safeStore1 -> {
+                if (valid.isEmpty()) // we've lost ownership of the range
+                    return AsyncResults.success(Ranges.EMPTY);
+                return fetch = safeStore1.dataStore().fetch(node, safeStore1, valid, syncPoint, this);
+            }))).flatMap(i -> i).begin(this);
         }
 
         // we no longer want to fetch these ranges (perhaps we no longer own them)
@@ -379,7 +371,7 @@ class Bootstrap
         }
 
         @Override
-        public void accept(Void success, Throwable failure)
+        public void accept(Object success, Throwable failure)
         {
             if (completed)
                 return;
diff --git a/accord-core/src/main/java/accord/local/Cleanup.java b/accord-core/src/main/java/accord/local/Cleanup.java
index baf96cb0..578912a6 100644
--- a/accord-core/src/main/java/accord/local/Cleanup.java
+++ b/accord-core/src/main/java/accord/local/Cleanup.java
@@ -68,6 +68,16 @@ public enum Cleanup
     }
 
     public static Cleanup shouldCleanup(SafeCommandStore safeStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+    {
+        return shouldCleanup(safeStore.commandStore(), command, toEpoch, maybeFullRoute);
+    }
+
+    public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute)
+    {
+        return shouldCleanup(commandStore, command, toEpoch, maybeFullRoute, true);
+    }
+
+    public static Cleanup shouldCleanup(CommandStore commandStore, Command command, EpochSupplier toEpoch, Unseekables<?> maybeFullRoute, boolean enforceInvariants)
     {
         if (command.saveStatus() == Erased)
             return Cleanup.NO; // once erased we no longer have executeAt, and may consider that a transaction is owned by us that should not be (as its txnId is an earlier epoch than we adopted a range)
@@ -85,7 +95,7 @@ public enum Cleanup
             toEpoch = executeAt;
 
         return shouldCleanup(command.txnId(), command.status(), command.durability(), toEpoch, route,
-                               safeStore.commandStore().redundantBefore(), safeStore.commandStore().durableBefore());
+                             commandStore.redundantBefore(), commandStore.durableBefore(), enforceInvariants);
     }
 
     public static Cleanup shouldCleanup(TxnId txnId, Status status, Durability durability, EpochSupplier toEpoch, Route<?> route, RedundantBefore redundantBefore, DurableBefore durableBefore)
@@ -124,7 +134,7 @@ public enum Cleanup
                 //      - we can impose additional validations here IF we receive an epoch upper bound
                 //      - we should be more robust to the presence/absence of executeAt
                 //      - be cognisant of future epochs that participated only for PreAccept/Accept, but where txn was not committed to execute in the epoch (this is why we provide null toEpoch here)
-                illegalState("Command " + txnId + " that is being loaded is not owned by this shard on route " + route);
+                illegalState("Command %s that is being loaded is not owned by this shard on route %s. Redundant before: %s", txnId, route, redundantBefore);
             }
         }
         switch (redundant)
diff --git a/accord-core/src/main/java/accord/local/Command.java b/accord-core/src/main/java/accord/local/Command.java
index 47b1c51c..ddc4b5a8 100644
--- a/accord-core/src/main/java/accord/local/Command.java
+++ b/accord-core/src/main/java/accord/local/Command.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import javax.annotation.Nullable;
 
 import accord.api.Key;
 import accord.api.Result;
@@ -54,8 +55,6 @@ import accord.utils.IndexedTriConsumer;
 import accord.utils.Invariants;
 import accord.utils.SimpleBitSet;
 
-import javax.annotation.Nullable;
-
 import static accord.local.Command.AbstractCommand.validate;
 import static accord.local.SaveStatus.AcceptedInvalidate;
 import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
@@ -341,7 +340,7 @@ public abstract class Command implements CommonAttributes
                         Invariants.checkState(result != null, "Result is null");
                         break;
                     case Invalidated:
-                        Invariants.checkState(validate.durability().isMaybeInvalidated());
+                        Invariants.checkState(validate.durability().isMaybeInvalidated(), "%s is not invalidated", validate.durability());
                     case Unknown:
                         Invariants.checkState(validate.durability() != Local);
                     case Erased:
diff --git a/accord-core/src/main/java/accord/local/CommandStore.java b/accord-core/src/main/java/accord/local/CommandStore.java
index f993974b..6c15cbc7 100644
--- a/accord-core/src/main/java/accord/local/CommandStore.java
+++ b/accord-core/src/main/java/accord/local/CommandStore.java
@@ -18,29 +18,6 @@
 
 package accord.local;
 
-import accord.api.LocalListeners;
-import accord.api.ProgressLog;
-import accord.api.DataStore;
-import accord.api.VisibleForImplementationTesting;
-import accord.coordinate.CollectCalculatedDeps;
-import accord.local.Command.WaitingOn;
-
-import javax.annotation.Nullable;
-import accord.api.Agent;
-
-import accord.local.CommandStores.RangesForEpoch;
-import accord.primitives.KeyDeps;
-import accord.primitives.Keys;
-import accord.primitives.Range;
-import accord.primitives.Routables;
-import accord.utils.async.AsyncChain;
-
-import accord.api.ConfigurationService.EpochReady;
-import accord.utils.DeterministicIdentitySet;
-import accord.utils.Invariants;
-import accord.utils.ReducingRangeMap;
-import accord.utils.async.AsyncResult;
-
 import java.util.AbstractMap.SimpleImmutableEntry;
 import java.util.Collections;
 import java.util.HashMap;
@@ -54,17 +31,39 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import javax.annotation.Nullable;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSortedMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import accord.api.Agent;
+import accord.api.ConfigurationService.EpochReady;
+import accord.api.DataStore;
+import accord.api.LocalListeners;
+import accord.api.ProgressLog;
+import accord.coordinate.CollectCalculatedDeps;
+import accord.local.Command.WaitingOn;
+import accord.local.CommandStores.RangesForEpoch;
+import accord.primitives.Deps;
 import accord.primitives.FullRoute;
+import accord.primitives.KeyDeps;
+import accord.primitives.Keys;
 import accord.primitives.Participants;
+import accord.primitives.Range;
 import accord.primitives.RangeDeps;
 import accord.primitives.Ranges;
+import accord.primitives.Routables;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
+import accord.utils.DeterministicIdentitySet;
+import accord.utils.Invariants;
+import accord.utils.ReducingRangeMap;
+import accord.utils.async.AsyncChain;
+import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 import org.agrona.collections.Int2ObjectHashMap;
 
@@ -75,6 +74,7 @@ import static accord.local.PreLoadContext.empty;
 import static accord.primitives.AbstractRanges.UnionMode.MERGE_ADJACENT;
 import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
+import static accord.utils.Invariants.checkState;
 import static accord.utils.Invariants.illegalState;
 
 /**
@@ -82,6 +82,8 @@ import static accord.utils.Invariants.illegalState;
  */
 public abstract class CommandStore implements AgentExecutor
 {
+    public static final Logger logger = LoggerFactory.getLogger(CommandStore.class);
+
     static class EpochUpdate
     {
         final RangesForEpoch newRangesForEpoch;
@@ -110,13 +112,13 @@ public abstract class CommandStore implements AgentExecutor
         // TODO (desired): can better encapsulate by accepting only the newRangesForEpoch and deriving the add/remove ranges
         public void add(long epoch, RangesForEpoch newRangesForEpoch, Ranges addRanges)
         {
-            RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
+            RedundantBefore addRedundantBefore = RedundantBefore.create(addRanges, epoch, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.minForEpoch(epoch));
             update(newRangesForEpoch, addRedundantBefore);
         }
 
         public void remove(long epoch, RangesForEpoch newRangesForEpoch, Ranges removeRanges)
         {
-            RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE);
+            RedundantBefore addRedundantBefore = RedundantBefore.create(removeRanges, Long.MIN_VALUE, epoch, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE);
             update(newRangesForEpoch, addRedundantBefore);
         }
 
@@ -150,9 +152,9 @@ public abstract class CommandStore implements AgentExecutor
     protected final LocalListeners listeners;
     protected final EpochUpdateHolder epochUpdateHolder;
 
-    // TODO (expected): schedule regular pruning of these collections
-    // bootstrapBeganAt and shardDurableAt are both canonical data sets mostly used for debugging / constructing
-    private NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
+    // Used in markShardStale to make sure the staleness includes in progresss bootstraps
+    private transient NavigableMap<TxnId, Ranges> bootstrapBeganAt = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY); // additive (i.e. once inserted, rolled-over until invalidated, and the floor entry contains additions)
+
     private RedundantBefore redundantBefore = RedundantBefore.EMPTY;
     // TODO (expected): store this only once per node
     private DurableBefore durableBefore = DurableBefore.EMPTY;
@@ -179,7 +181,13 @@ public abstract class CommandStore implements AgentExecutor
     private final Set<Bootstrap> bootstraps = Collections.synchronizedSet(new DeterministicIdentitySet<>());
     @Nullable private ReducingRangeMap<Timestamp> rejectBefore;
 
-    protected CommandStore(int id, NodeTimeService time, Agent agent, DataStore store, ProgressLog.Factory progressLogFactory, LocalListeners.Factory listenersFactory, EpochUpdateHolder epochUpdateHolder)
+    protected CommandStore(int id,
+                           NodeTimeService time,
+                           Agent agent,
+                           DataStore store,
+                           ProgressLog.Factory progressLogFactory,
+                           LocalListeners.Factory listenersFactory,
+                           EpochUpdateHolder epochUpdateHolder)
     {
         this.id = id;
         this.time = time;
@@ -201,20 +209,19 @@ public abstract class CommandStore implements AgentExecutor
         return agent;
     }
 
-    public RangesForEpoch updateRangesForEpoch()
+    public void updateRangesForEpoch(SafeCommandStore safeStore)
     {
         EpochUpdate update = epochUpdateHolder.get();
         if (update == null)
-            return rangesForEpoch;
+            return;
 
         update = epochUpdateHolder.getAndSet(null);
         if (!update.addGlobalRanges.isEmpty())
-            setDurableBefore(DurableBefore.merge(durableBefore, DurableBefore.create(update.addGlobalRanges, TxnId.NONE, TxnId.NONE)));
+            safeStore.upsertDurableBefore(DurableBefore.create(update.addGlobalRanges, TxnId.NONE, TxnId.NONE));
         if (update.addRedundantBefore.size() > 0)
-            setRedundantBefore(RedundantBefore.merge(redundantBefore, update.addRedundantBefore));
+            safeStore.upsertRedundantBefore(update.addRedundantBefore);
         if (update.newRangesForEpoch != null)
-            rangesForEpoch = update.newRangesForEpoch;
-        return rangesForEpoch;
+            safeStore.setRangesForEpoch(update.newRangesForEpoch);
     }
 
     public RangesForEpoch unsafeRangesForEpoch()
@@ -222,6 +229,11 @@ public abstract class CommandStore implements AgentExecutor
         return rangesForEpoch;
     }
 
+    protected void unsafeSetRangesForEpoch(RangesForEpoch newRangesForEpoch)
+    {
+        rangesForEpoch = newRangesForEpoch;
+    }
+
     public abstract boolean inStore();
 
     public void maybeExecuteImmediately(Runnable task)
@@ -235,43 +247,45 @@ public abstract class CommandStore implements AgentExecutor
     public abstract <T> AsyncChain<T> submit(PreLoadContext context, Function<? super SafeCommandStore, T> apply);
     public abstract void shutdown();
 
-    // implementations are expected to override this for persistence
-    protected void setRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
+    protected abstract void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore);
+
+    protected void upsertDurableBefore(DurableBefore addDurableBefore)
+    {
+        durableBefore = DurableBefore.merge(durableBefore, addDurableBefore);
+    }
+
+    protected void unsafeSetRejectBefore(ReducingRangeMap<Timestamp> newRejectBefore)
     {
         this.rejectBefore = newRejectBefore;
     }
 
-    /**
-     * To be overridden by implementations, to ensure the new state is persisted
-     *
-     * TODO (required): consider handling asynchronicity of persistence
-     *  (could leave to impls to call this parent method once persisted)
-     * TODO (desired): compact Ranges, merging overlaps
-     */
-    protected void setBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
+    // Should be called _only_ via safe command store
+    protected void upsertRedundantBefore(RedundantBefore addRedundantBefore)
     {
-        this.bootstrapBeganAt = newBootstrapBeganAt;
+        redundantBefore = RedundantBefore.merge(redundantBefore, addRedundantBefore);
     }
 
-    public DurableBefore durableBefore()
+    protected void unsafeSetDurableBefore(DurableBefore newDurableBefore)
     {
-        return durableBefore;
+        durableBefore = newDurableBefore;
     }
 
-    /**
-     * To be overridden by implementations, to ensure the new state is persisted.
-     */
-    public void setDurableBefore(DurableBefore durableBefore)
+    protected void unsafeSetRedundantBefore(RedundantBefore newRedundantBefore)
     {
-        this.durableBefore = durableBefore;
+        redundantBefore = newRedundantBefore;
     }
 
     /**
-     * To be overridden by implementations, to ensure the new state is persisted.
+     * This method may be invoked on a non-CommandStore thread
      */
-    protected void setRedundantBefore(RedundantBefore newRedundantBefore)
+    protected synchronized void unsafeSetSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
+    {
+        this.safeToRead = newSafeToRead;
+    }
+
+    protected void unsafeSetBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
     {
-        this.redundantBefore = newRedundantBefore;
+        this.bootstrapBeganAt = newBootstrapBeganAt;
     }
 
     /**
@@ -293,29 +307,21 @@ public abstract class CommandStore implements AgentExecutor
         setMaxConflicts(maxConflicts.update(keysOrRanges, executeAt));
     }
 
-    /**
-     * This method may be invoked on a non-CommandStore thread
-     */
-    protected synchronized void setSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
-    {
-        this.safeToRead = newSafeToRead;
-    }
-
     public final void markExclusiveSyncPoint(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
     {
         // TODO (desired): narrow ranges to those that are owned
         Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
         ReducingRangeMap<Timestamp> newRejectBefore = rejectBefore != null ? rejectBefore : new ReducingRangeMap<>();
         newRejectBefore = ReducingRangeMap.add(newRejectBefore, ranges, txnId, Timestamp::max);
-        setRejectBefore(newRejectBefore);
+        unsafeSetRejectBefore(newRejectBefore);
     }
 
     public final void markExclusiveSyncPointLocallyApplied(SafeCommandStore safeStore, TxnId txnId, Ranges ranges)
     {
         // TODO (desired): narrow ranges to those that are owned
         Invariants.checkArgument(txnId.kind() == ExclusiveSyncPoint);
-        RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE));
-        setRedundantBefore(newRedundantBefore);
+        RedundantBefore newRedundantBefore = RedundantBefore.merge(redundantBefore, RedundantBefore.create(ranges, txnId, TxnId.NONE, TxnId.NONE, TxnId.NONE));
+        unsafeSetRedundantBefore(newRedundantBefore);
         updatedRedundantBefore(safeStore, txnId, ranges);
     }
 
@@ -334,10 +340,7 @@ public abstract class CommandStore implements AgentExecutor
             return time.uniqueNow(txnId).asRejected();
 
         if (txnId.kind() == ExclusiveSyncPoint)
-        {
-            markExclusiveSyncPoint(safeStore, txnId, (Ranges)keys);
             return txnId;
-        }
 
         // TODO (expected): reject if any transaction exists with a higher timestamp OR a higher epoch
         //   this permits us to agree fast path decisions across epoch changes
@@ -475,7 +478,7 @@ public abstract class CommandStore implements AgentExecutor
             }
             else
             {
-                // TODO (correcness) : PreLoadContext only works with Seekables, which doesn't allow mixing Keys and Ranges... But Deps has both Keys AND Ranges!
+                // TODO (correctness) : PreLoadContext only works with Seekables, which doesn't allow mixing Keys and Ranges... But Deps has both Keys AND Ranges!
                 // ATM all known implementations store ranges in-memory, but this will not be true soon, so this will need to be addressed
                 execute(contextFor(null, deps.txnIds(), deps.keyDeps.keys(), COMMANDS), safeStore -> {
                     safeStore.registerHistoricalTransactions(deps);
@@ -511,23 +514,36 @@ public abstract class CommandStore implements AgentExecutor
 
     final void markBootstrapping(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
     {
-        setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
-        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId);
-        setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
-        DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
-        setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
+        safeStore.setBootstrapBeganAt(bootstrap(globalSyncId, ranges, bootstrapBeganAt));
+        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, TxnId.NONE, globalSyncId);
+        safeStore.upsertRedundantBefore(addRedundantBefore);
+        safeStore.upsertDurableBefore(DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE));
+        // TODO: can we use `upsert` for notifications?
         updatedRedundantBefore(safeStore, globalSyncId, ranges);
     }
 
     // TODO (expected): we can immediately truncate dependencies locally once an exclusiveSyncPoint applies, we don't need to wait for the whole shard
-    public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges ranges)
-    {
-        ranges = ranges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
-        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE);
-        setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
-        DurableBefore addDurableBefore = DurableBefore.create(ranges, globalSyncId, globalSyncId);
-        setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
-        updatedRedundantBefore(safeStore, globalSyncId, ranges);
+    public void markShardDurable(SafeCommandStore safeStore, TxnId globalSyncId, Ranges durableRanges)
+    {
+        final Ranges slicedRanges = durableRanges.slice(safeStore.ranges().allUntil(globalSyncId.epoch()), Minimal);
+        RedundantBefore addShardRedundant = RedundantBefore.create(slicedRanges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, globalSyncId, TxnId.NONE, TxnId.NONE);
+        safeStore.upsertRedundantBefore(addShardRedundant);
+        DurableBefore addDurableBefore = DurableBefore.create(slicedRanges, globalSyncId, globalSyncId);
+        safeStore.upsertDurableBefore(addDurableBefore);
+        updatedRedundantBefore(safeStore, globalSyncId, slicedRanges);
+        safeStore = safeStore; // make unusable in lambda
+        safeStore.dataStore().snapshot(slicedRanges, globalSyncId).begin((success, fail) -> {
+            if (fail != null)
+            {
+                logger.error("Unsuccessful dataStore snapshot; unable to update GC markers", fail);
+                return;
+            }
+
+            execute(PreLoadContext.empty(), safeStore0 -> {
+                RedundantBefore addGc = RedundantBefore.create(slicedRanges, Long.MIN_VALUE, Long.MAX_VALUE, TxnId.NONE, TxnId.NONE, globalSyncId, TxnId.NONE);
+                safeStore0.upsertRedundantBefore(addGc);
+            });
+        });
     }
 
     protected void updatedRedundantBefore(SafeCommandStore safeStore, TxnId syncId, Ranges ranges)
@@ -552,21 +568,39 @@ public abstract class CommandStore implements AgentExecutor
         }
         agent.onStale(staleSince, ranges);
 
-        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
-        setRedundantBefore(RedundantBefore.merge(redundantBefore, addRedundantBefore));
+        RedundantBefore addRedundantBefore = RedundantBefore.create(ranges, TxnId.NONE, TxnId.NONE, TxnId.NONE, TxnId.NONE, staleUntilAtLeast);
+        safeStore.upsertRedundantBefore(addRedundantBefore);
         // find which ranges need to bootstrap, subtracting those already in progress that cover the id
 
         markUnsafeToRead(ranges);
     }
 
     // MUST be invoked before CommandStore reference leaks to anyone
+    // The integration may have already loaded persisted values for these fields before this is called
+    // so it must be a merge for each field with the initialization values. These starting values don't need to be
+    // persisted since we can synthesize them at startup every time
+    // TODO (review): This needs careful thought about not persisting and that purgeAndInsert is doing the right thing
+    // with safeToRead
     Supplier<EpochReady> initialise(long epoch, Ranges ranges)
     {
-        DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
-        setDurableBefore(DurableBefore.merge(durableBefore, addDurableBefore));
-        setBootstrapBeganAt(ImmutableSortedMap.of(TxnId.NONE, ranges));
-        setSafeToRead(ImmutableSortedMap.of(Timestamp.NONE, ranges));
-        return () -> new EpochReady(epoch, DONE, DONE, DONE, DONE);
+        return () -> {
+            AsyncResult<Void> done = execute(empty(), (safeStore) -> {
+                // Merge in a base for any ranges that needs to be covered
+                DurableBefore addDurableBefore = DurableBefore.create(ranges, TxnId.NONE, TxnId.NONE);
+                safeStore.upsertDurableBefore(addDurableBefore);
+                // TODO (review): Convoluted check to not overwrite existing bootstraps with TxnId.NONE
+                // If loading from disk didn't finish before this then we might initialize the range at TxnId.NONE?
+                // Does CommandStores.topology ensure that doesn't happen? Is it fine if it does because it will get superseded?
+                Ranges newBootstrapRanges = ranges;
+                for (Ranges existing : bootstrapBeganAt.values())
+                    newBootstrapRanges = newBootstrapRanges.without(existing);
+                if (!newBootstrapRanges.isEmpty())
+                    bootstrapBeganAt = bootstrap(TxnId.NONE, newBootstrapRanges, bootstrapBeganAt);
+                safeStore.setSafeToRead(purgeAndInsert(safeToRead, TxnId.NONE, ranges));
+            }).beginAsResult();
+
+            return new EpochReady(epoch, DONE, DONE, DONE, DONE);
+        };
     }
 
     public final Ranges safeToReadAt(Timestamp at)
@@ -585,10 +619,15 @@ public abstract class CommandStore implements AgentExecutor
         return redundantBefore;
     }
 
-    @VisibleForImplementationTesting
-    public NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
+    public DurableBefore durableBefore()
+    {
+        return durableBefore;
+    }
+
+    @VisibleForTesting
+    public final NavigableMap<TxnId, Ranges> bootstrapBeganAt() { return bootstrapBeganAt; }
 
-    @VisibleForImplementationTesting
+    @VisibleForTesting
     public NavigableMap<Timestamp, Ranges> safeToRead() { return safeToRead; }
 
     public final boolean isRejectedIfNotPreAccepted(TxnId txnId, Unseekables<?> participants)
@@ -739,21 +778,30 @@ public abstract class CommandStore implements AgentExecutor
         return redundantBefore.status(minimumDependencyId, executeAt, participantsOfWaitingTxn).compareTo(RedundantStatus.PARTIALLY_PRE_BOOTSTRAP_OR_STALE) >= 0;
     }
 
-    final synchronized void markUnsafeToRead(Ranges ranges)
+    final void markUnsafeToRead(Ranges ranges)
     {
         if (safeToRead.values().stream().anyMatch(r -> r.intersects(ranges)))
-            setSafeToRead(purgeHistory(safeToRead, ranges));
+        {
+            execute(empty(), safeStore -> {
+                safeStore.setSafeToRead(purgeHistory(safeToRead, ranges));
+            }).beginAsResult();
+        }
     }
 
     final synchronized void markSafeToRead(Timestamp forBootstrapAt, Timestamp at, Ranges ranges)
     {
-        Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
-        setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
+        execute(empty(), safeStore -> {
+            Ranges validatedSafeToRead = redundantBefore.validateSafeToRead(forBootstrapAt, ranges);
+            safeStore.setSafeToRead(purgeAndInsert(safeToRead, at, validatedSafeToRead));
+        }).beginAsResult();
     }
 
-    private static <T extends Timestamp> ImmutableSortedMap<T, Ranges> bootstrap(T at, Ranges ranges, NavigableMap<T, Ranges> bootstrappedAt)
+    public static ImmutableSortedMap<TxnId, Ranges> bootstrap(TxnId at, Ranges ranges, NavigableMap<TxnId, Ranges> bootstrappedAt)
     {
-        Invariants.checkArgument(bootstrappedAt.lastKey().compareTo(at) < 0);
+        Invariants.checkArgument(bootstrappedAt.lastKey().compareTo(at) < 0 || at == TxnId.NONE);
+        if (at == TxnId.NONE)
+            for (Ranges rs : bootstrappedAt.values())
+                checkState(!ranges.intersects(rs));
         Invariants.checkArgument(!ranges.isEmpty());
         // if we're bootstrapping these ranges, then any period we previously owned the ranges for is effectively invalidated
         return purgeAndInsert(bootstrappedAt, at, ranges);
diff --git a/accord-core/src/main/java/accord/local/CommandStores.java b/accord-core/src/main/java/accord/local/CommandStores.java
index 9d6660ff..545a7843 100644
--- a/accord-core/src/main/java/accord/local/CommandStores.java
+++ b/accord-core/src/main/java/accord/local/CommandStores.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -29,8 +30,11 @@ import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import javax.annotation.Nonnull;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import accord.api.Agent;
 import accord.api.ConfigurationService.EpochReady;
@@ -56,14 +60,9 @@ import accord.utils.MapReduceConsume;
 import accord.utils.RandomSource;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
-import javax.annotation.Nonnull;
-
 import org.agrona.collections.Hashing;
 import org.agrona.collections.Int2ObjectHashMap;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import static accord.api.ConfigurationService.EpochReady.done;
 import static accord.local.PreLoadContext.empty;
 import static accord.primitives.Routables.Slice.Minimal;
@@ -143,6 +142,33 @@ public abstract class CommandStores
     // TODO (expected): merge with RedundantBefore, and standardise executeRanges() to treat removing stale ranges the same as adding new epoch ranges
     public static class RangesForEpoch
     {
+        public static class Snapshot
+        {
+            public final long[] epochs;
+            public final Ranges[] ranges;
+
+            public Snapshot(long[] epochs, Ranges[] ranges)
+            {
+                this.epochs = epochs;
+                this.ranges = ranges;
+            }
+
+            @Override
+            public boolean equals(Object o)
+            {
+                if (this == o) return true;
+                if (o == null || getClass() != o.getClass()) return false;
+                Snapshot snapshot = (Snapshot) o;
+                return Objects.deepEquals(epochs, snapshot.epochs) && Objects.deepEquals(ranges, snapshot.ranges);
+            }
+
+            @Override
+            public int hashCode()
+            {
+                return Objects.hash(Arrays.hashCode(epochs), Arrays.hashCode(ranges));
+            }
+        }
+
         final long[] epochs;
         final Ranges[] ranges;
         final CommandStore store;
@@ -161,6 +187,11 @@ public abstract class CommandStores
             this.store = store;
         }
 
+        public Snapshot snapshot()
+        {
+            return new Snapshot(epochs, ranges);
+        }
+
         public RangesForEpoch withRanges(long epoch, Ranges latestRanges)
         {
             Invariants.checkArgument(epochs.length == 0 || epochs[epochs.length - 1] <= epoch);
@@ -690,6 +721,15 @@ public abstract class CommandStores
         return shards[supplier.random.nextInt(shards.length)].store;
     }
 
+    public CommandStore[] all()
+    {
+        ShardHolder[] shards = current.shards;
+        CommandStore[] all = new CommandStore[shards.length];
+        for (int i = 0; i < shards.length; i++)
+            all[i] = shards[i].store;
+        return all;
+    }
+
     public CommandStore forId(int id)
     {
         Snapshot snapshot = current;
diff --git a/accord-core/src/main/java/accord/local/Commands.java b/accord-core/src/main/java/accord/local/Commands.java
index 43b8d77f..a20d81ba 100644
--- a/accord-core/src/main/java/accord/local/Commands.java
+++ b/accord-core/src/main/java/accord/local/Commands.java
@@ -75,9 +75,7 @@ import static accord.local.Status.PreApplied;
 import static accord.local.Status.PreCommitted;
 import static accord.local.Status.Stable;
 import static accord.local.Status.Truncated;
-import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Route.isFullRoute;
-import static accord.primitives.Txn.Kind.ExclusiveSyncPoint;
 import static accord.utils.Invariants.illegalState;
 
 public class Commands
@@ -547,16 +545,23 @@ public class Commands
         return safeStore.ranges().applyRanges(executeAt);
     }
 
-    private static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
+    public static AsyncChain<Void> applyChain(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
     {
         Command.Executed command = safeStore.get(txnId).current().asExecuted();
         if (command.hasBeen(Applied))
             return AsyncChains.success(null);
+        return apply(safeStore, context, txnId);
+    }
 
+    public static AsyncChain<Void> apply(SafeCommandStore safeStore, PreLoadContext context, TxnId txnId)
+    {
+        Command.Executed command = safeStore.get(txnId).current().asExecuted();
         // TODO (required): make sure we are correctly handling (esp. C* side with validation logic) executing a transaction
         //  that was pre-bootstrap for some range (so redundant and we may have gone ahead of), but had to be executed locally
         //  for another range
         CommandStore unsafeStore = safeStore.commandStore();
+        // TODO (required, API): do we care about tracking the write persistence latency, when this is just a memtable write?
+        //  the only reason it will be slow is because Memtable flushes are backed-up (which will be reported elsewhere)
         long t0 = safeStore.time().now();
         return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()), command.partialTxn())
                .flatMap(unused -> unsafeStore.submit(context, ss -> {
@@ -568,7 +573,23 @@ public class Commands
                }));
     }
 
-    private static void apply(SafeCommandStore safeStore, Command.Executed command)
+    public static AsyncChain<Void> applyWrites(SafeCommandStore safeStore, PreLoadContext context, Command command)
+    {
+        CommandStore unsafeStore = safeStore.commandStore();
+        Ranges executeRanges = executeRanges(safeStore, command.executeAt());
+        Command.Executed executed = command.asExecuted();
+        boolean intersects = executed.writes().keys.intersects(executeRanges);
+        if (intersects)
+            return command.writes().apply(safeStore, applyRanges(safeStore, command.executeAt()), command.partialTxn())
+                          .flatMap(unused -> unsafeStore.submit(context, ss -> {
+                              postApply(ss, command.txnId());
+                              return null;
+                          }));
+        else
+            return AsyncChains.success(null);
+    }
+
+    public static void apply(SafeCommandStore safeStore, Command command)
     {
         CommandStore unsafeStore = safeStore.commandStore();
         TxnId txnId = command.txnId();
@@ -588,9 +609,13 @@ public class Commands
         }
     }
 
-    private static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
+    public static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
+    {
+        return maybeExecute(safeStore, safeCommand, safeCommand.current(), alwaysNotifyListeners, notifyWaitingOn);
+    }
+
+    public static boolean maybeExecute(SafeCommandStore safeStore, SafeCommand safeCommand, Command command, boolean alwaysNotifyListeners, boolean notifyWaitingOn)
     {
-        final Command command = safeCommand.current();
         if (logger.isTraceEnabled())
             logger.trace("{}: Maybe executing with status {}. Will notify listeners on noop: {}", command.txnId(), command.status(), alwaysNotifyListeners);
 
@@ -614,7 +639,6 @@ public class Commands
 
         // TODO (required): slice our execute ranges based on any pre-bootstrap state
         // FIXME: need to communicate to caller that we didn't execute if we take one of the above paths
-
         switch (command.status())
         {
             case Stable:
@@ -647,12 +671,6 @@ public class Commands
                     //      but: if we later support transitive dependency elision this could be dangerous
                     logger.trace("{}: applying no-op", command.txnId());
                     safeCommand.applied(safeStore);
-                    if (command.txnId().kind() == ExclusiveSyncPoint)
-                    {
-                        Ranges ranges = safeStore.ranges().allAt(command.txnId().epoch());
-                        ranges = command.route().slice(ranges, Minimal).participants().toRanges();
-                        safeStore.commandStore().markExclusiveSyncPointLocallyApplied(safeStore, command.txnId(), ranges);
-                    }
                     safeStore.notifyListeners(safeCommand, command);
                     return true;
                 }
@@ -851,6 +869,16 @@ public class Commands
                               || (command.route() == null || Infer.safeToCleanup(safeStore, command, command.route(), command.executeAt()) || safeStore.isFullyPreBootstrapOrStale(command, command.route().participants()))
         , "Command %s could not be truncated", command);
 
+        Command result = purge(command, maybeFullRoute, cleanup);
+        safeCommand.update(safeStore, result);
+        safeStore.progressLog().clear(safeCommand.txnId());
+        if (notifyListeners)
+            safeStore.notifyListeners(safeCommand, result);
+        return result;
+    }
+
+    public static Command purge(Command command, @Nullable Unseekables<?> maybeFullRoute, Cleanup cleanup)
+    {
         Command result;
         switch (cleanup)
         {
@@ -861,14 +889,14 @@ public class Commands
                 break;
 
             case TRUNCATE_WITH_OUTCOME:
-                Invariants.checkArgument(!command.hasBeen(Truncated));
+                Invariants.checkArgument(!command.hasBeen(Truncated), "%s", command);
                 Invariants.checkState(command.hasBeen(PreApplied));
                 result = truncatedApplyWithOutcome(command.asExecuted());
                 break;
 
             case TRUNCATE:
-                Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0);
-                Invariants.checkState(command.hasBeen(PreApplied));
+                Invariants.checkState(command.saveStatus().compareTo(TruncatedApply) < 0, "%s should not have been in %s but is %s", command, TruncatedApply, command.saveStatus());
+                Invariants.checkState(command.hasBeen(PreApplied),"%s should have been %s", command, PreApplied);
                 result = truncatedApply(command, Route.tryCastToFullRoute(maybeFullRoute));
                 break;
 
@@ -877,11 +905,6 @@ public class Commands
                 result = erased(command);
                 break;
         }
-
-        safeCommand.update(safeStore, result);
-        safeStore.progressLog().clear(safeCommand.txnId());
-        if (notifyListeners)
-            safeStore.notifyListeners(safeCommand, command);
         return result;
     }
 
@@ -1160,7 +1183,7 @@ public class Commands
     public static Command updateRouteOrParticipants(SafeCommandStore safeStore, SafeCommand safeCommand, Unseekables<?> participants)
     {
         Command current = safeCommand.current();
-        if (current.hasBeen(Invalidated))
+        if (current.saveStatus().compareTo(Erased) >= 0)
             return current;
 
         CommonAttributes updated = current;
diff --git a/accord-core/src/main/java/accord/local/DurableBefore.java b/accord-core/src/main/java/accord/local/DurableBefore.java
index 6e485f38..aab40559 100644
--- a/accord-core/src/main/java/accord/local/DurableBefore.java
+++ b/accord-core/src/main/java/accord/local/DurableBefore.java
@@ -42,6 +42,11 @@ public class DurableBefore extends ReducingRangeMap<DurableBefore.Entry>
     {
         public static DurableBefore create(boolean inclusiveEnds, RoutingKey[] ends, Entry[] values)
         {
+            if (values.length == 0)
+            {
+                Invariants.checkState(ends.length == 1 && ends[0] == null);
+                return DurableBefore.EMPTY;
+            }
             return new DurableBefore(inclusiveEnds, ends, values);
         }
     }
diff --git a/accord-core/src/main/java/accord/local/PreLoadContext.java b/accord-core/src/main/java/accord/local/PreLoadContext.java
index 25c68efd..d5153a79 100644
--- a/accord-core/src/main/java/accord/local/PreLoadContext.java
+++ b/accord-core/src/main/java/accord/local/PreLoadContext.java
@@ -138,23 +138,58 @@ public interface PreLoadContext
     static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
     {
         Invariants.checkState(!additional.contains(primary));
-        return new PreLoadContext()
+        return new Standard(primary, additional, keys, keyHistory);
+    }
+
+    class Standard implements PreLoadContext
+    {
+        private final TxnId primary;
+        private final Collection<TxnId> additional;
+        private final Seekables<?, ?> keys;
+        private final KeyHistory keyHistory;
+
+        public Standard(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys, KeyHistory keyHistory)
         {
-            @Override
-            public TxnId primaryTxnId()
-            {
-                return primary;
-            }
+            this.primary = primary;
+            this.additional = additional;
+            this.keys = keys;
+            this.keyHistory = keyHistory;
+        }
 
-            @Override
-            public Collection<TxnId> additionalTxnIds() { return additional; }
+        public String toString()
+        {
+            return "PreLoadContext{" +
+                   "primary=" + primary +
+                   ", additional=" + additional +
+                   ", keys=" + keys +
+                   ", keyHistory=" + keyHistory +
+                   '}';
+        }
 
-            @Override
-            public Seekables<?, ?> keys() { return keys; }
+        @Override
+        @Nullable
+        public TxnId primaryTxnId()
+        {
+            return primary;
+        }
 
-            @Override
-            public KeyHistory keyHistory() { return keyHistory; }
-        };
+        @Override
+        public Collection<TxnId> additionalTxnIds()
+        {
+            return additional;
+        }
+
+        @Override
+        public Seekables<?, ?> keys()
+        {
+            return keys;
+        }
+
+        @Override
+        public KeyHistory keyHistory()
+        {
+            return keyHistory;
+        }
     }
 
     static PreLoadContext contextFor(TxnId primary, Collection<TxnId> additional, Seekables<?, ?> keys)
diff --git a/accord-core/src/main/java/accord/local/RedundantBefore.java b/accord-core/src/main/java/accord/local/RedundantBefore.java
index 26942bf6..a39cdf98 100644
--- a/accord-core/src/main/java/accord/local/RedundantBefore.java
+++ b/accord-core/src/main/java/accord/local/RedundantBefore.java
@@ -95,6 +95,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
          */
         public final @Nonnull TxnId shardAppliedOrInvalidatedBefore;
 
+        /**
+         * Represents the maximum TxnId we know to have fully executed until across all healthy replicas for the range in question.
+         * Unless we are stale or pre-bootstrap, in which case no such guarantees can be made.
+         */
+        public final @Nonnull TxnId gcBefore;
+
         /**
          * bootstrappedAt defines the txnId bounds we expect to maintain data for locally.
          *
@@ -116,12 +122,12 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
          */
         public final @Nullable Timestamp staleUntilAtLeast;
 
-        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
         {
-            this(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+            this(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
-        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+        public Entry(Range range, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId locallyDecidedAndAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
         {
             this.range = range;
             this.startEpoch = startEpoch;
@@ -129,11 +135,14 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             this.locallyAppliedOrInvalidatedBefore = locallyAppliedOrInvalidatedBefore;
             this.locallyDecidedAndAppliedOrInvalidatedBefore = locallyDecidedAndAppliedOrInvalidatedBefore;
             this.shardAppliedOrInvalidatedBefore = shardAppliedOrInvalidatedBefore;
+            this.gcBefore = gcBefore;
             this.bootstrappedAt = bootstrappedAt;
             this.staleUntilAtLeast = staleUntilAtLeast;
             Invariants.checkArgument(locallyAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyAppliedOrInvalidatedBefore.domain().isRange());
             Invariants.checkArgument(locallyDecidedAndAppliedOrInvalidatedBefore.equals(TxnId.NONE) || locallyDecidedAndAppliedOrInvalidatedBefore.domain().isRange());
             Invariants.checkArgument(shardAppliedOrInvalidatedBefore.equals(TxnId.NONE) || shardAppliedOrInvalidatedBefore.domain().isRange());
+            Invariants.checkArgument(gcBefore.equals(TxnId.NONE) || gcBefore.domain().isRange());
+            Invariants.checkArgument(gcBefore.compareTo(shardAppliedOrInvalidatedBefore) <= 0);
         }
 
         public static Entry reduce(Entry a, Entry b)
@@ -154,6 +163,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             int cl = cur.locallyAppliedOrInvalidatedBefore.compareTo(add.locallyAppliedOrInvalidatedBefore);
             int cd = cur.locallyDecidedAndAppliedOrInvalidatedBefore.compareTo(add.locallyDecidedAndAppliedOrInvalidatedBefore);
             int cs = cur.shardAppliedOrInvalidatedBefore.compareTo(add.shardAppliedOrInvalidatedBefore);
+            int cg = cur.gcBefore.compareTo(add.gcBefore);
             int cb = cur.bootstrappedAt.compareTo(add.bootstrappedAt);
             int csu = compareStaleUntilAtLeast(cur.staleUntilAtLeast, add.staleUntilAtLeast);
 
@@ -165,6 +175,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             TxnId locallyAppliedOrInvalidatedBefore = cl >= 0 ? cur.locallyAppliedOrInvalidatedBefore : add.locallyAppliedOrInvalidatedBefore;
             TxnId locallyDecidedAndAppliedOrInvalidatedBefore = cd >= 0 ? cur.locallyDecidedAndAppliedOrInvalidatedBefore : add.locallyDecidedAndAppliedOrInvalidatedBefore;
             TxnId shardAppliedOrInvalidatedBefore = cs >= 0 ? cur.shardAppliedOrInvalidatedBefore : add.shardAppliedOrInvalidatedBefore;
+            TxnId gcBefore = cg >= 0 ? cur.gcBefore : add.gcBefore;
             TxnId bootstrappedAt = cb >= 0 ? cur.bootstrappedAt : add.bootstrappedAt;
             Timestamp staleUntilAtLeast = csu >= 0 ? cur.staleUntilAtLeast : add.staleUntilAtLeast;
 
@@ -182,7 +193,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             if (staleUntilAtLeast != null && bootstrappedAt.compareTo(staleUntilAtLeast) >= 0)
                 staleUntilAtLeast = null;
 
-            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, locallyDecidedAndAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         static @Nonnull RedundantStatus getAndMerge(Entry entry, @Nonnull RedundantStatus prev, TxnId txnId, EpochSupplier executeAt)
@@ -305,7 +316,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
 
         Entry withRange(Range range)
         {
-            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+            return new Entry(range, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         }
 
         public boolean equals(Object that)
@@ -376,27 +387,27 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
         return Ranges.ofSortedAndDeoverlapped(staleRanges).mergeTouching();
     }
 
-    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
     {
-        return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+        return create(ranges, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
     }
 
-    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+    public static RedundantBefore create(Ranges ranges, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
     {
-        return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+        return create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
     }
 
-    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt)
+    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt)
     {
-        return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, null);
+        return create(ranges, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, null);
     }
 
-    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
+    public static RedundantBefore create(Ranges ranges, long startEpoch, long endEpoch, @Nonnull TxnId locallyAppliedOrInvalidatedBefore, @Nonnull TxnId shardAppliedOrInvalidatedBefore, @Nonnull TxnId gcBefore, @Nonnull TxnId bootstrappedAt, @Nullable Timestamp staleUntilAtLeast)
     {
         if (ranges.isEmpty())
             return new RedundantBefore();
 
-        Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, bootstrappedAt, staleUntilAtLeast);
+        Entry entry = new Entry(null, startEpoch, endEpoch, locallyAppliedOrInvalidatedBefore, shardAppliedOrInvalidatedBefore, gcBefore, bootstrappedAt, staleUntilAtLeast);
         Builder builder = new Builder(ranges.get(0).endInclusive(), ranges.size() * 2);
         for (int i = 0 ; i < ranges.size() ; ++i)
         {
@@ -509,7 +520,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             if (v.range.start().equals(start) && v.range.end().equals(end))
                 return v;
 
-            return new Entry(v.range.newRange(start, end), v.startEpoch, v.endEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.bootstrappedAt, v.staleUntilAtLeast);
+            return new Entry(v.range.newRange(start, end), v.startEpoch, v.endEpoch, v.locallyAppliedOrInvalidatedBefore, v.shardAppliedOrInvalidatedBefore, v.gcBefore, v.bootstrappedAt, v.staleUntilAtLeast);
         }
 
         @Override
@@ -528,7 +539,7 @@ public class RedundantBefore extends ReducingRangeMap<RedundantBefore.Entry>
             return new Entry(a.range.newRange(
                 a.range.start().compareTo(b.range.start()) <= 0 ? a.range.start() : b.range.start(),
                 a.range.end().compareTo(b.range.end()) >= 0 ? a.range.end() : b.range.end()
-            ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.bootstrappedAt, a.staleUntilAtLeast);
+            ), a.startEpoch, a.endEpoch, a.locallyAppliedOrInvalidatedBefore, a.shardAppliedOrInvalidatedBefore, a.gcBefore, a.bootstrappedAt, a.staleUntilAtLeast);
         }
 
         @Override
diff --git a/accord-core/src/main/java/accord/local/SafeCommandStore.java b/accord-core/src/main/java/accord/local/SafeCommandStore.java
index e74e29af..cc51f7c8 100644
--- a/accord-core/src/main/java/accord/local/SafeCommandStore.java
+++ b/accord-core/src/main/java/accord/local/SafeCommandStore.java
@@ -18,6 +18,7 @@
 
 package accord.local;
 
+import java.util.NavigableMap;
 import javax.annotation.Nullable;
 
 import accord.api.Agent;
@@ -34,10 +35,12 @@ import accord.primitives.EpochSupplier;
 import accord.primitives.Keys;
 import accord.primitives.Participants;
 import accord.primitives.Ranges;
+import accord.primitives.Routable.Domain;
 import accord.primitives.Routables;
 import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.Timestamp;
+import accord.primitives.Txn.Kind;
 import accord.primitives.Txn.Kind.Kinds;
 import accord.primitives.TxnId;
 import accord.primitives.Unseekables;
@@ -46,8 +49,10 @@ import accord.utils.Invariants;
 import static accord.local.Cleanup.NO;
 import static accord.local.KeyHistory.COMMANDS;
 import static accord.local.RedundantBefore.PreBootstrapOrStale.FULLY;
+import static accord.local.SaveStatus.Applied;
 import static accord.local.SaveStatus.Erased;
 import static accord.local.SaveStatus.ErasedOrInvalidOrVestigial;
+import static accord.primitives.Routables.Slice.Minimal;
 import static accord.primitives.Route.isFullRoute;
 
 /**
@@ -206,9 +211,32 @@ public abstract class SafeCommandStore
     {
         updateMaxConflicts(prev, updated);
         updateCommandsForKey(prev, updated);
+        updateExclusiveSyncPoint(prev, updated);
     }
 
-    private void updateMaxConflicts(Command prev, Command updated)
+    public void updateExclusiveSyncPoint(Command prev, Command updated)
+    {
+        if (updated.txnId().kind() != Kind.ExclusiveSyncPoint || updated.txnId().domain() != Domain.Range) return;
+        if (updated.route() == null) return;
+
+        SaveStatus oldSaveStatus = prev == null ? SaveStatus.Uninitialised : prev.saveStatus();
+        SaveStatus newSaveStatus = updated.saveStatus();
+
+        TxnId txnId = updated.txnId();
+        if (newSaveStatus.known.isDefinitionKnown() && !oldSaveStatus.known.isDefinitionKnown())
+        {
+            Ranges ranges = updated.route().slice(ranges().all(), Minimal).toRanges();
+            commandStore().markExclusiveSyncPoint(this, txnId, ranges);
+        }
+
+        if (newSaveStatus == Applied && oldSaveStatus != Applied)
+        {
+            Ranges ranges = updated.route().slice(ranges().all(), Minimal).toRanges();
+            commandStore().markExclusiveSyncPointLocallyApplied(this, txnId, ranges);
+        }
+    }
+
+    public void updateMaxConflicts(Command prev, Command updated)
     {
         SaveStatus oldSaveStatus = prev == null ? SaveStatus.Uninitialised : prev.saveStatus();
         SaveStatus newSaveStatus = updated.saveStatus();
@@ -222,7 +250,41 @@ public abstract class SafeCommandStore
         commandStore().updateMaxConflicts(prev, updated);
     }
 
-    private void updateCommandsForKey(Command prev, Command next)
+    /**
+     * Methods that implementors can use to capture changes to auxiliary collections:
+     */
+
+    public void upsertRedundantBefore(RedundantBefore addRedundantBefore)
+    {
+        commandStore().upsertRedundantBefore(addRedundantBefore);
+    }
+
+    public void upsertDurableBefore(DurableBefore addDurableBefore)
+    {
+        commandStore().upsertDurableBefore(addDurableBefore);
+    }
+
+    public void setBootstrapBeganAt(NavigableMap<TxnId, Ranges> newBootstrapBeganAt)
+    {
+        commandStore().unsafeSetBootstrapBeganAt(newBootstrapBeganAt);
+    }
+
+    public void setSafeToRead(NavigableMap<Timestamp, Ranges> newSafeToRead)
+    {
+        commandStore().unsafeSetSafeToRead(newSafeToRead);
+    }
+
+    public void setRangesForEpoch(CommandStores.RangesForEpoch rangesForEpoch)
+    {
+        commandStore().unsafeSetRangesForEpoch(rangesForEpoch);
+    }
+
+    protected void registerHistoricalTransactions(Deps deps)
+    {
+        commandStore().registerHistoricalTransactions(deps, this);
+    }
+
+    public void updateCommandsForKey(Command prev, Command next)
     {
         if (!CommandsForKey.needsUpdate(prev, next))
             return;
@@ -311,8 +373,6 @@ public abstract class SafeCommandStore
     public abstract NodeTimeService time();
     public abstract CommandStores.RangesForEpoch ranges();
 
-    public abstract void registerHistoricalTransactions(Deps deps);
-
     public boolean isTruncated(Command command)
     {
         return Cleanup.shouldCleanup(this, command, null, null) != NO;
diff --git a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
index eb5137b7..7cc2bb84 100644
--- a/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
+++ b/accord-core/src/main/java/accord/local/cfk/CommandsForKey.java
@@ -193,6 +193,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
 {
     private static final Logger logger = LoggerFactory.getLogger(CommandsForKey.class);
 
+    private static boolean reportLinearizabilityViolations = true;
     private static final boolean ELIDE_TRANSITIVE_DEPENDENCIES = true;
 
     public static final TxnInfo NO_INFO = new TxnInfo(TxnId.NONE, HISTORICAL, TxnId.NONE);
@@ -1577,7 +1578,7 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
             for (int i = maxAppliedWriteByExecuteAt ; i >= 0 ; --i)
             {
                 TxnInfo txn = committedByExecuteAt[i];
-                if (newInfo == txn)
+                if (newInfo == txn && CommandsForKey.reportLinearizabilityViolations())
                 {
                     // we haven't found anything pre-bootstrap that follows this command, so log a linearizability violation
                     // TODO (expected): this should be a rate-limited logger; need to integrate with Cassandra
@@ -1693,4 +1694,19 @@ public class CommandsForKey extends CommandsForKeyUpdate implements CommandsSumm
             }
         }
     }
+
+    public static boolean reportLinearizabilityViolations()
+    {
+        return reportLinearizabilityViolations;
+    }
+
+    public static void enableLinearizabilityViolationsReporting()
+    {
+        reportLinearizabilityViolations = true;
+    }
+
+    public static void disableLinearizabilityViolationsReporting()
+    {
+        reportLinearizabilityViolations = false;
+    }
 }
diff --git a/accord-core/src/main/java/accord/local/cfk/NotifySink.java b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
index 91909849..5a306890 100644
--- a/accord-core/src/main/java/accord/local/cfk/NotifySink.java
+++ b/accord-core/src/main/java/accord/local/cfk/NotifySink.java
@@ -39,8 +39,6 @@ interface NotifySink
 {
     void notWaiting(SafeCommandStore safeStore, TxnId txnId, Key key);
 
-    void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key);
-
     void waitingOn(SafeCommandStore safeStore, TxnInfo txn, Key key, SaveStatus waitingOnStatus, BlockedUntil blockedUntil, boolean notifyCfk);
 
     class DefaultNotifySink implements NotifySink
@@ -60,8 +58,7 @@ interface NotifySink
             }
         }
 
-        @Override
-        public void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
+        private void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
         {
             Commands.removeWaitingOnKeyAndMaybeExecute(safeStore, safeCommand, key);
         }
diff --git a/accord-core/src/main/java/accord/local/cfk/Updating.java b/accord-core/src/main/java/accord/local/cfk/Updating.java
index 2939b5c7..fdf8eb9d 100644
--- a/accord-core/src/main/java/accord/local/cfk/Updating.java
+++ b/accord-core/src/main/java/accord/local/cfk/Updating.java
@@ -53,6 +53,7 @@ import static accord.local.cfk.CommandsForKey.InternalStatus.TRANSITIVELY_KNOWN;
 import static accord.local.cfk.CommandsForKey.managesExecution;
 import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.APPLY;
 import static accord.local.cfk.CommandsForKey.Unmanaged.Pending.COMMIT;
+import static accord.local.cfk.CommandsForKey.reportLinearizabilityViolations;
 import static accord.local.cfk.Pruning.loadPruned;
 import static accord.local.cfk.Pruning.loadingPrunedFor;
 import static accord.local.cfk.Utils.insertMissing;
@@ -615,7 +616,7 @@ class Updating
                     {
                         for (int i = pos; i <= maxAppliedWriteByExecuteAt; ++i)
                         {
-                            if (committedByExecuteAt[pos].kind().witnesses(newInfo))
+                            if (committedByExecuteAt[pos].kind().witnesses(newInfo) && reportLinearizabilityViolations())
                                 logger.error("Linearizability violation on key {}: {} is committed to execute (at {}) before {} that should witness it but has already applied (at {})", cfk.key, newInfo.plainTxnId(), newInfo.plainExecuteAt(), committedByExecuteAt[i].plainTxnId(), committedByExecuteAt[i].plainExecuteAt());
                         }
                     }
@@ -673,7 +674,7 @@ class Updating
             TxnInfo[] committedByExecuteAt = cfk.committedByExecuteAt;
             for (int i = cfk.maxAppliedWriteByExecuteAt + 1; i < appliedPos ; ++i)
             {
-                if (committedByExecuteAt[i].status != APPLIED && appliedKind.witnesses(committedByExecuteAt[i]))
+                if (committedByExecuteAt[i].status != APPLIED && appliedKind.witnesses(committedByExecuteAt[i]) && reportLinearizabilityViolations())
                     logger.error("Linearizability violation on key {}: {} is committed to execute (at {}) before {} that should witness it but has already applied (at {})", cfk.key, committedByExecuteAt[i].plainTxnId(), committedByExecuteAt[i].plainExecuteAt(), applied.plainTxnId(), applied.plainExecuteAt());
             }
         }
diff --git a/accord-core/src/main/java/accord/messages/PreAccept.java b/accord-core/src/main/java/accord/messages/PreAccept.java
index 10af97c2..7d7d514f 100644
--- a/accord-core/src/main/java/accord/messages/PreAccept.java
+++ b/accord-core/src/main/java/accord/messages/PreAccept.java
@@ -115,6 +115,7 @@ public class PreAccept extends WithUnsynced<PreAccept.PreAcceptReply> implements
         //      must consider cleanup though. (alternative is to make recovery more complicated)
         Ranges ranges = safeStore.ranges().allBetween(minEpoch, txnId);
         if (txnId.kind() == ExclusiveSyncPoint)
+            // TODO (required): this must be removed for correctly restoring rejectBefore (so journal can update it). This is expected to happen in a follow-up patch soon after.
             safeStore.commandStore().markExclusiveSyncPoint(safeStore, txnId, ranges);
         return new PreAcceptOk(txnId, txnId, calculatePartialDeps(safeStore, txnId, partialTxn.keys(), scope, EpochSupplier.constant(minEpoch), txnId, ranges));
     }
diff --git a/accord-core/src/main/java/accord/messages/SetGloballyDurable.java b/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
index 3fa39321..7d912435 100644
--- a/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
+++ b/accord-core/src/main/java/accord/messages/SetGloballyDurable.java
@@ -18,8 +18,8 @@
 
 package accord.messages;
 
-import accord.local.PreLoadContext;
 import accord.local.DurableBefore;
+import accord.local.PreLoadContext;
 import accord.local.SafeCommandStore;
 import accord.primitives.TxnId;
 
@@ -51,7 +51,8 @@ public class SetGloballyDurable extends AbstractEpochRequest<SimpleReply>
     {
         DurableBefore cur = safeStore.commandStore().durableBefore();
         DurableBefore upd = DurableBefore.merge(durableBefore, cur);
-        safeStore.commandStore().setDurableBefore(upd);
+        // This is done asynchronously
+        safeStore.upsertDurableBefore(upd);
         return Ok;
     }
 
diff --git a/accord-core/src/main/java/accord/primitives/Writes.java b/accord-core/src/main/java/accord/primitives/Writes.java
index e470d074..cdc76efb 100644
--- a/accord-core/src/main/java/accord/primitives/Writes.java
+++ b/accord-core/src/main/java/accord/primitives/Writes.java
@@ -77,6 +77,10 @@ public class Writes
             accumulate.add(write.apply(key, safeStore, executeAt, safeStore.dataStore(), txn));
             return accumulate;
         }, new ArrayList<>());
+
+        if (futures.isEmpty())
+            return SUCCESS;
+
         return AsyncChains.reduce(futures, (l, r) -> null);
     }
 
diff --git a/accord-core/src/main/java/accord/topology/Topologies.java b/accord-core/src/main/java/accord/topology/Topologies.java
index 95fce506..2d4a3494 100644
--- a/accord-core/src/main/java/accord/topology/Topologies.java
+++ b/accord-core/src/main/java/accord/topology/Topologies.java
@@ -18,12 +18,15 @@
 
 package accord.topology;
 
+import java.util.AbstractCollection;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
-import com.google.common.collect.Sets;
+import com.google.common.collect.Iterators;
 
 import accord.api.TopologySorter;
 import accord.local.Node;
@@ -74,7 +77,29 @@ public interface Topologies extends TopologySorter
     // note this can be expensive to evaluate
     SortedList<Id> nodes();
 
-    Set<Node.Id> nonStaleNodes();
+    default Collection<Node.Id> nonStaleNodes()
+    {
+        Topology currentTopology = current();
+        Set<Id> staleIds = currentTopology.staleIds();
+        SortedList<Id> nodes = nodes();
+        if (staleIds.isEmpty())
+            return nodes;
+
+        return new AbstractCollection<>()
+        {
+            @Override
+            public Iterator<Id> iterator()
+            {
+                return Iterators.filter(nodes.iterator(), id -> !staleIds.contains(id));
+            }
+
+            @Override
+            public int size()
+            {
+                return Iterators.size(iterator());
+            }
+        };
+    }
 
     int estimateUniqueNodes();
 
@@ -216,12 +241,6 @@ public interface Topologies extends TopologySorter
             return topology.nodes();
         }
 
-        @Override
-        public Set<Node.Id> nonStaleNodes()
-        {
-            return topology.nonStaleNodes();
-        }
-
         @Override
         public int estimateUniqueNodes()
         {
@@ -402,13 +421,6 @@ public interface Topologies extends TopologySorter
             return result;
         }
 
-        @Override
-        public Set<Node.Id> nonStaleNodes()
-        {
-            Topology currentTopology = current();
-            return Sets.filter(nodes(), id -> !currentTopology.staleIds().contains(id));
-        }
-
         @Override
         public Ranges computeRangesForNode(Id node)
         {
diff --git a/accord-core/src/main/java/accord/topology/Topology.java b/accord-core/src/main/java/accord/topology/Topology.java
index f1fc988f..fca2618b 100644
--- a/accord-core/src/main/java/accord/topology/Topology.java
+++ b/accord-core/src/main/java/accord/topology/Topology.java
@@ -34,9 +34,6 @@ import java.util.stream.IntStream;
 import javax.annotation.Nullable;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Sets;
-import org.agrona.collections.Int2ObjectHashMap;
-import org.agrona.collections.IntArrayList;
 
 import accord.api.RoutingKey;
 import accord.local.Node.Id;
@@ -53,6 +50,8 @@ import accord.utils.IndexedTriFunction;
 import accord.utils.SimpleBitSet;
 import accord.utils.SortedArrays.SortedArrayList;
 import accord.utils.Utils;
+import org.agrona.collections.Int2ObjectHashMap;
+import org.agrona.collections.IntArrayList;
 
 import static accord.utils.Invariants.illegalArgument;
 import static accord.utils.SortedArrays.Search.FLOOR;
@@ -558,11 +557,6 @@ public class Topology
         return nodeIds;
     }
 
-    public Set<Id> nonStaleNodes()
-    {
-        return Sets.filter(nodes(), id -> !staleIds.contains(id));
-    }
-
     public Ranges ranges()
     {
         return subsetOfRanges;
diff --git a/accord-core/src/main/java/accord/utils/LogGroupTimers.java b/accord-core/src/main/java/accord/utils/LogGroupTimers.java
index 1245c03c..1c3251b6 100644
--- a/accord-core/src/main/java/accord/utils/LogGroupTimers.java
+++ b/accord-core/src/main/java/accord/utils/LogGroupTimers.java
@@ -132,6 +132,15 @@ public class LogGroupTimers<T extends LogGroupTimers.Timer>
             if (scheduled.isDone() || newDeadline < scheduledAt - schedulerPreciseLateTolerance)
                 ensureScheduled(now);
         }
+
+        public void clear()
+        {
+            if (!scheduled.isDone())
+                scheduled.cancel();
+            scheduled = Scheduler.CANCELLED;
+            lastNow = 0;
+            scheduledAt = Long.MAX_VALUE;
+        }
     }
 
     static class Bucket<T extends Timer> extends IntrusivePriorityHeap<T> implements Comparable<Bucket<T>>
@@ -604,4 +613,16 @@ public class LogGroupTimers<T extends LogGroupTimers.Timer>
         for (int i = bucketsStart + 1 ; i < bucketsEnd ; ++i)
             Invariants.checkState(buckets[i - 1].end() == buckets[i].epoch);
     }
+
+    public void clear()
+    {
+        while (bucketsStart < bucketsEnd)
+        {
+            buckets[bucketsStart].clear();
+            buckets[bucketsStart++] = null;
+        }
+        bucketsStart = bucketsEnd = 0;
+        curEpoch = 0;
+        addFinger = null;
+    }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java b/accord-core/src/main/java/accord/utils/TriConsumer.java
similarity index 72%
copy from accord-core/src/test/java/accord/impl/basic/PendingQueue.java
copy to accord-core/src/main/java/accord/utils/TriConsumer.java
index fd4932d4..72d1ec03 100644
--- a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
+++ b/accord-core/src/main/java/accord/utils/TriConsumer.java
@@ -16,17 +16,9 @@
  * limitations under the License.
  */
 
-package accord.impl.basic;
+package accord.utils;
 
-import java.util.concurrent.TimeUnit;
-
-public interface PendingQueue
+public interface TriConsumer<P1, P2, P3>
 {
-    void add(Pending item);
-    void addNoDelay(Pending item);
-    void add(Pending item, long delay, TimeUnit units);
-    boolean remove(Pending item);
-    Pending poll();
-    int size();
-    long nowInMillis();
+    void consume(P1 p1, P2 p2, P3 p3);
 }
diff --git a/accord-core/src/main/java/accord/utils/async/AsyncChain.java b/accord-core/src/main/java/accord/utils/async/AsyncChain.java
index 66f97f16..a642321b 100644
--- a/accord-core/src/main/java/accord/utils/async/AsyncChain.java
+++ b/accord-core/src/main/java/accord/utils/async/AsyncChain.java
@@ -21,6 +21,7 @@ package accord.utils.async;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
+import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
@@ -48,6 +49,16 @@ public interface AsyncChain<V>
         return AsyncChains.flatMap(this, mapper, executor);
     }
 
+    default <T> AsyncChain<T> map(Function<? super V, ? extends T> mapper, BooleanSupplier inExecutor, Executor executor)
+    {
+        return flatMap(input -> {
+            if (inExecutor.getAsBoolean())
+                return AsyncChains.success(mapper.apply(input));
+            else
+                return AsyncChains.ofCallable(executor, () -> mapper.apply(input));
+        });
+    }
+
     /**
      * When the chain has failed, this allows the chain to attempt to recover if possible.  The provided function may return a {@code null} to represent
      * that recovery was not possible and that the original exception should propgate.
diff --git a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
index 19f18426..664f025a 100644
--- a/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
+++ b/accord-core/src/test/java/accord/impl/RemoteListenersTest.java
@@ -31,7 +31,6 @@ import java.util.function.Function;
 import java.util.function.IntSupplier;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
-
 import javax.annotation.Nullable;
 
 import org.junit.jupiter.api.Assertions;
@@ -384,7 +383,14 @@ public class RemoteListenersTest
 
         protected TestCommandStore(int id)
         {
-            super(id, null, null, null, ignore -> new ProgressLog.NoOpProgressLog(), ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultLocalListeners.DefaultNotifySink.INSTANCE), new EpochUpdateHolder());
+            super(id,
+                  null,
+                  null,
+                  null,
+                  ignore -> new ProgressLog.NoOpProgressLog(),
+                  ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}),
+                                                      DefaultLocalListeners.DefaultNotifySink.INSTANCE),
+                  new EpochUpdateHolder());
             this.storeId = id;
         }
 
@@ -392,6 +398,8 @@ public class RemoteListenersTest
         @Override public AsyncChain<Void> execute(PreLoadContext context, Consumer<? super SafeCommandStore> consumer) { return null; }
         @Override public <T> AsyncChain<T> submit(PreLoadContext context, Function<? super SafeCommandStore, T> apply) { return null; }
         @Override public void shutdown() {}
+        @Override protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore) {}
+
         @Override public <T> AsyncChain<T> submit(Callable<T> task) { return null; }
     }
 
@@ -418,6 +426,5 @@ public class RemoteListenersTest
         @Override public ProgressLog progressLog() { return null; }
         @Override public NodeTimeService time() { return null; }
         @Override public CommandStores.RangesForEpoch ranges() { return null; }
-        @Override public void registerHistoricalTransactions(Deps deps) { }
     }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/Cluster.java b/accord-core/src/test/java/accord/impl/basic/Cluster.java
index b24f161a..a938052e 100644
--- a/accord-core/src/test/java/accord/impl/basic/Cluster.java
+++ b/accord-core/src/test/java/accord/impl/basic/Cluster.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -41,7 +42,6 @@ import java.util.function.IntSupplier;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
-
 import javax.annotation.Nullable;
 
 import org.junit.jupiter.api.Assertions;
@@ -50,12 +50,12 @@ import org.slf4j.LoggerFactory;
 
 import accord.api.BarrierType;
 import accord.api.Key;
+import accord.api.LocalConfig;
 import accord.api.MessageSink;
 import accord.api.Scheduler;
 import accord.burn.BurnTestConfigurationService;
 import accord.burn.TopologyUpdates;
 import accord.burn.random.FrequentLargeRange;
-import accord.api.LocalConfig;
 import accord.coordinate.Barrier;
 import accord.coordinate.CoordinationAdapter;
 import accord.coordinate.Exhausted;
@@ -64,22 +64,23 @@ import accord.coordinate.Preempted;
 import accord.coordinate.Timeout;
 import accord.coordinate.Truncated;
 import accord.impl.CoordinateDurabilityScheduling;
+import accord.impl.DefaultLocalListeners;
+import accord.impl.DefaultRemoteListeners;
 import accord.impl.DefaultRequestTimeouts;
 import accord.impl.InMemoryCommandStore.GlobalCommand;
 import accord.impl.MessageListener;
 import accord.impl.PrefixedIntHashKey;
-import accord.impl.DefaultLocalListeners;
-import accord.impl.progresslog.DefaultProgressLogs;
-import accord.impl.DefaultRemoteListeners;
 import accord.impl.SizeOfIntersectionSorter;
 import accord.impl.TopologyFactory;
 import accord.impl.basic.DelayedCommandStores.DelayedCommandStore;
 import accord.impl.list.ListAgent;
 import accord.impl.list.ListStore;
+import accord.impl.progresslog.DefaultProgressLogs;
 import accord.local.AgentExecutor;
 import accord.local.Command;
-import accord.local.Node.Id;
+import accord.local.CommandStore;
 import accord.local.Node;
+import accord.local.Node.Id;
 import accord.local.NodeTimeService;
 import accord.local.SaveStatus;
 import accord.local.ShardDistributor;
@@ -219,21 +220,54 @@ public class Cluster implements Scheduler
             processNext(next);
     }
 
+    boolean hasNonRecurring()
+    {
+        boolean hasNonRecurring = false;
+        for (Pending p : pending)
+        {
+            if (!(p instanceof RecurringPendingRunnable))
+                continue;
+            RecurringPendingRunnable r = (RecurringPendingRunnable) p;
+            if (r.requeue.hasNonRecurring())
+            {
+                hasNonRecurring = true;
+                break;
+            }
+        }
+
+        return hasNonRecurring;
+    }
+
     public boolean processPending()
     {
         checkFailures.run();
-        if (pending.size() == recurring)
+        // All remaining tasks are recurring
+        if (recurring > 0 && pending.size() == recurring && !hasNonRecurring())
             return false;
 
-        Object next = pending.poll();
+        Pending next = pending.poll();
         if (next == null)
             return false;
 
         processNext(next);
+
         checkFailures.run();
         return true;
     }
 
+    /**
+     * Drain tasks that match predicate.
+     *
+     * Returns whether any tasks were processed
+     */
+    public boolean drain(Predicate<Pending> process)
+    {
+        List<Pending> pending = this.pending.drain(process);
+        for (Pending p : pending)
+            processNext(p);
+        return !pending.isEmpty();
+    }
+
     private void processNext(Object next)
     {
         if (next instanceof Packet)
@@ -257,7 +291,8 @@ public class Cluster implements Scheduler
                     else callback.success(deliver.src, reply);
                 }
             }
-            else journalLookup.apply(deliver.dst).handle((Request) deliver.message, deliver.src, deliver);
+
+            else on.receive((Request) deliver.message, deliver.src, deliver);
         }
         else
         {
@@ -274,21 +309,27 @@ public class Cluster implements Scheduler
     @Override
     public Scheduled recurring(Runnable run, long delay, TimeUnit units)
     {
-        RecurringPendingRunnable result = new RecurringPendingRunnable(pending, run, delay, units);
-        ++recurring;
-        result.onCancellation(() -> --recurring);
-        pending.add(result, delay, units);
-        return result;
+        return recurring(run, () -> delay, units);
     }
 
     @Override
     public Scheduled once(Runnable run, long delay, TimeUnit units)
     {
-        RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, delay, units);
+        RecurringPendingRunnable result = new RecurringPendingRunnable(null, run, () -> delay, units);
         pending.add(result, delay, units);
         return result;
     }
 
+    public Scheduled recurring(Runnable run, LongSupplier delay, TimeUnit units)
+    {
+        RecurringPendingRunnable result = new RecurringPendingRunnable(pending, run, delay, units);
+        ++recurring;
+        result.onCancellation(() -> --recurring);
+        pending.add(result, delay.getAsLong(), units);
+        return result;
+
+    }
+
     public void onDone(Runnable run)
     {
         onDone.add(run);
@@ -422,12 +463,12 @@ public class Cluster implements Scheduler
                 BiConsumer<Timestamp, Ranges> onStale = (sinceAtLeast, ranges) -> configRandomizer.onStale(id, sinceAtLeast, ranges);
                 AgentExecutor nodeExecutor = nodeExecutorSupplier.apply(id, onStale);
                 executorMap.put(id, nodeExecutor);
-                Journal journal = new Journal();
+                Journal journal = new Journal(id);
                 journalMap.put(id, journal);
                 BurnTestConfigurationService configService = new BurnTestConfigurationService(id, nodeExecutor, randomSupplier, topology, nodeMap::get, topologyUpdates);
                 BooleanSupplier isLoadedCheck = Gens.supplier(Gens.bools().mixedDistribution().next(random), random);
                 Node node = new Node(id, messageSink, configService, nowSupplier, NodeTimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.MILLISECONDS, nowSupplier),
-                                     () -> new ListStore(id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
+                                     () -> new ListStore(sinks, random, id), new ShardDistributor.EvenSplit<>(8, ignore -> new PrefixedIntHashKey.Splitter()),
                                      nodeExecutor.agent(),
                                      randomSupplier.get(), sinks, SizeOfIntersectionSorter.SUPPLIER, DefaultRemoteListeners::new, DefaultRequestTimeouts::new,
                                      DefaultProgressLogs::new, DefaultLocalListeners.Factory::new, DelayedCommandStores.factory(sinks.pending, isLoadedCheck, journal), new CoordinationAdapter.DefaultFactory(),
@@ -461,8 +502,6 @@ public class Cluster implements Scheduler
             updateDurabilityRate.run();
             schemaApply.onUpdate(topology);
 
-            // startup
-            journalMap.entrySet().forEach(e -> e.getValue().start(nodeMap.get(e.getKey())));
             AsyncResult<?> startup = AsyncChains.reduce(nodeMap.values().stream().map(Node::unsafeStart).collect(toList()), (a, b) -> null).beginAsResult();
             while (sinks.processPending());
             Assertions.assertTrue(startup.isDone());
@@ -475,11 +514,54 @@ public class Cluster implements Scheduler
             }, 5L, SECONDS);
 
             Scheduled reconfigure = sinks.recurring(configRandomizer::maybeUpdateTopology, 1, SECONDS);
+
+            Scheduled purge = sinks.recurring(() -> {
+                trace.debug("Triggering purge.");
+                int numNodes = random.nextInt(1, nodeMap.size());
+                for (int i = 0; i < numNodes; i++)
+                {
+                    Id id = random.pick(nodes);
+                    Node node = nodeMap.get(id);
+
+                    Journal journal = journalMap.get(node.id());
+                    CommandStore[] stores = nodeMap.get(node.id()).commandStores().all();
+                    journal.purge((j) -> stores[j]);
+                }
+            }, () -> random.nextInt(1, 10), SECONDS);
+
+            Scheduled restart = sinks.recurring(() -> {
+                Id id = random.pick(nodes);
+                CommandStore[] stores = nodeMap.get(id).commandStores().all();
+                Predicate<Pending> pred = getPendingPredicate(stores);
+                while (sinks.drain(pred));
+
+                // Journal cleanup is a rough equivalent of a node restart.
+                trace.debug("Triggering journal cleanup for node " + id);
+                CommandsForKey.disableLinearizabilityViolationsReporting();
+                ListStore listStore = (ListStore) nodeMap.get(id).commandStores().dataStore();
+                listStore.clear();
+                listStore.restoreFromSnapshot();
+
+                Journal journal = journalMap.get(id);
+                for (CommandStore s : stores)
+                {
+                    DelayedCommandStores.DelayedCommandStore store = (DelayedCommandStores.DelayedCommandStore) s;
+                    store.clearForTesting();
+                    journal.reconstructAll(store.loader(), store.id());
+                    journal.loadHistoricalTransactions(store::load, store.id());
+                }
+                while (sinks.drain(pred));
+                CommandsForKey.enableLinearizabilityViolationsReporting();
+                trace.debug("Done with cleanup.");
+            }, () -> random.nextInt(1, 10), SECONDS);
+
             durabilityScheduling.forEach(CoordinateDurabilityScheduling::start);
             services.forEach(Service::start);
 
             noMoreWorkSignal.accept(() -> {
                 reconfigure.cancel();
+                purge.cancel();
+                restart.cancel();
                 durabilityScheduling.forEach(CoordinateDurabilityScheduling::stop);
                 services.forEach(Service::close);
             });
@@ -517,11 +599,24 @@ public class Cluster implements Scheduler
         }
         finally
         {
-            journalMap.values().forEach(Journal::shutdown);
             nodeMap.values().forEach(Node::shutdown);
         }
     }
 
+    private static Predicate<Pending> getPendingPredicate(CommandStore[] stores)
+    {
+        Set<CommandStore> nodeStores = new HashSet<>(Arrays.asList(stores));
+        return item -> {
+            if (item instanceof DelayedCommandStore.DelayedTask)
+            {
+                DelayedCommandStore.DelayedTask<?> task = (DelayedCommandStore.DelayedTask<?>) item;
+                if (nodeStores.contains(task.parent()))
+                    return true;
+            }
+            return false;
+        };
+    }
+
     private interface Service extends AutoCloseable
     {
         void start();
diff --git a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
index 6e4de714..0de6debc 100644
--- a/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
+++ b/accord-core/src/test/java/accord/impl/basic/DelayedCommandStores.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.concurrent.Callable;
 import java.util.function.BiConsumer;
@@ -32,11 +33,13 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import accord.api.Agent;
 import accord.api.DataStore;
 import accord.api.LocalListeners;
 import accord.api.ProgressLog;
-import accord.api.Result;
 import accord.impl.InMemoryCommandStore;
 import accord.impl.InMemoryCommandStores;
 import accord.impl.InMemorySafeCommand;
@@ -44,7 +47,6 @@ import accord.impl.InMemorySafeCommandsForKey;
 import accord.impl.InMemorySafeTimestampsForKey;
 import accord.impl.PrefixedIntHashKey;
 import accord.impl.basic.TaskExecutorService.Task;
-import accord.impl.mock.MockStore;
 import accord.local.Command;
 import accord.local.CommandStore;
 import accord.local.CommandStores;
@@ -53,6 +55,7 @@ import accord.local.NodeTimeService;
 import accord.local.PreLoadContext;
 import accord.local.SafeCommandStore;
 import accord.local.ShardDistributor;
+import accord.primitives.Deps;
 import accord.primitives.Range;
 import accord.primitives.RoutableKey;
 import accord.primitives.Txn;
@@ -71,6 +74,8 @@ import static accord.utils.Invariants.ParanoiaCostFactor.HIGH;
 
 public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
 {
+    private static final Logger logger = LoggerFactory.getLogger(DelayedCommandStores.class);
+
     //TODO (correctness): remove once we have a Journal integration that does not change the values of a command...
     // There are known issues due to Journal, so exclude known problamtic fields
     private static class ValidationHack
@@ -138,13 +143,18 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
 
     public static class DelayedCommandStore extends InMemoryCommandStore
     {
-        private class DelayedTask<T> extends Task<T>
+        public class DelayedTask<T> extends Task<T>
         {
             private DelayedTask(Callable<T> fn)
             {
                 super(fn);
             }
 
+            public DelayedCommandStore parent()
+            {
+                return DelayedCommandStore.this;
+            }
+
             @Override
             public void run()
             {
@@ -175,11 +185,8 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
             if (current.txnId().kind() == Txn.Kind.EphemeralRead)
                 return;
 
-            Result result = current.result();
-            if (result == null)
-                result = MockStore.RESULT;
             // Journal will not have result persisted. This part is here for test purposes and ensuring that we have strict object equality.
-            Command reconstructed = journal.reconstruct(id, current.txnId(), result);
+            Command reconstructed = journal.reconstruct(id, current.txnId());
             List<Difference<?>> diff = ReflectionUtils.recursiveEquals(current, reconstructed);
             List<String> filteredDiff = diff.stream().filter(d -> !DelayedCommandStores.hasKnownIssue(d.path)).map(Object::toString).collect(Collectors.toList());
             Invariants.checkState(filteredDiff.isEmpty(), "Commands did not match: expected %s, given %s, node %s, store %d, diff %s", current, reconstructed, time, id(), new LazyToString(() -> String.join("\n", filteredDiff)));
@@ -272,6 +279,13 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
         {
             return new DelayedSafeStore(this, ranges, context, commands, timestampsForKey, commandsForKeys);
         }
+
+        @Override
+        protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+        {
+            journal.registerHistoricalTransactions(id(), deps);
+            super.registerHistoricalTransactions(deps, safeStore);
+        }
     }
 
     public static class DelayedSafeStore extends InMemoryCommandStore.InMemorySafeStore
@@ -286,16 +300,16 @@ public class DelayedCommandStores extends InMemoryCommandStores.SingleThread
         @Override
         public void postExecute()
         {
-            super.postExecute();
             commands.entrySet().forEach(e -> {
                 InMemorySafeCommand safe = e.getValue();
                 if (!safe.isModified()) return;
 
                 Command before = safe.original();
                 Command after = safe.current();
-                commandStore.journal.onExecute(commandStore.id(), before, after, context.primaryTxnId().equals(after.txnId()));
+                commandStore.journal.onExecute(commandStore.id(), before, after, Objects.equals(context.primaryTxnId(), after.txnId()));
                 commandStore.validateRead(safe.current());
             });
+            super.postExecute();
         }
     }
 
diff --git a/accord-core/src/test/java/accord/impl/basic/Journal.java b/accord-core/src/test/java/accord/impl/basic/Journal.java
index 28e37b99..e11a8dec 100644
--- a/accord-core/src/test/java/accord/impl/basic/Journal.java
+++ b/accord-core/src/test/java/accord/impl/basic/Journal.java
@@ -18,26 +18,33 @@
 
 package accord.impl.basic;
 
-import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.TimeUnit;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.IntFunction;
+import java.util.stream.Collectors;
 
 import accord.api.Result;
+import accord.impl.InMemoryCommandStore;
+import accord.local.Cleanup;
 import accord.local.Command;
+import accord.local.CommandStore;
+import accord.local.Commands;
 import accord.local.CommonAttributes;
 import accord.local.Node;
 import accord.local.SaveStatus;
 import accord.local.Status;
-import accord.messages.Message;
-import accord.messages.ReplyContext;
-import accord.messages.Request;
 import accord.primitives.Ballot;
+import accord.primitives.Deps;
 import accord.primitives.PartialDeps;
 import accord.primitives.PartialTxn;
 import accord.primitives.Route;
@@ -47,121 +54,132 @@ import accord.primitives.TxnId;
 import accord.primitives.Writes;
 import accord.utils.Invariants;
 import org.agrona.collections.Long2ObjectHashMap;
-import org.agrona.collections.LongArrayList;
 
+import static accord.local.Status.Invalidated;
+import static accord.local.Status.Truncated;
 import static accord.utils.Invariants.illegalState;
 
-
-public class Journal implements Runnable
+public class Journal
 {
-    private final Queue<RequestContext> unframedRequests = new ArrayDeque<>();
-    private final LongArrayList waitForEpochs = new LongArrayList();
-    private final Long2ObjectHashMap<ArrayList<RequestContext>> delayedRequests = new Long2ObjectHashMap<>();
-    private final Map<Key, List<Diff>> diffs = new HashMap<>();
-    private Node node;
-    boolean isScheduled;
-
-    public Journal()
-    {
-    }
+    private final Long2ObjectHashMap<NavigableMap<TxnId, List<Diff>>> diffsPerCommandStore = new Long2ObjectHashMap<>();
+    private final Map<Integer, List<Deps>> historicalTransactions = new HashMap<>();
 
-    public void start(Node node)
-    {
-        this.node = node;
-    }
+    private final Node.Id id;
 
-    private void ensureScheduled()
+    public Journal(Node.Id id)
     {
-        if (isScheduled) return;
-        node.scheduler().once(this, 1, TimeUnit.MILLISECONDS);
-        isScheduled = true;
+        this.id = id;
     }
 
-    public void shutdown()
+    public void purge(IntFunction<CommandStore> storeSupplier)
     {
-        this.node = null;
-    }
-
-    public void handle(Request request, Node.Id from, ReplyContext replyContext)
-    {
-        ensureScheduled();
-        if (request.type() != null && request.type().hasSideEffects())
+        for (Map.Entry<Long, NavigableMap<TxnId, List<Diff>>> e : diffsPerCommandStore.entrySet())
         {
-            // enqueue
-            unframedRequests.add(new RequestContext(request, request.waitForEpoch(), () -> node.receive(request, from, replyContext)));
-            return;
+            int commandStoreId = e.getKey().intValue();
+            Map<TxnId, List<Diff>> localJournal = e.getValue();
+            CommandStore store = storeSupplier.apply(commandStoreId);
+
+            Map<TxnId, List<Diff>> updates = new HashMap<>();
+            List<TxnId> removals = new ArrayList<>();
+            for (Map.Entry<TxnId, List<Diff>> e2 : localJournal.entrySet())
+            {
+                TxnId txnId = e2.getKey();
+                List<Diff> diffs = e2.getValue();
+                Command command = reconstruct(diffs, Reconstruct.Last).get(0);
+                if (command.status() == Truncated || command.status() == Invalidated)
+                    continue; // Already truncated
+                Cleanup cleanup = Cleanup.shouldCleanup(store, command, null, command.route(), false);
+                switch (cleanup)
+                {
+                    case NO:
+                        break;
+                    case INVALIDATE:
+                    case TRUNCATE_WITH_OUTCOME:
+                    case TRUNCATE:
+                        command = Commands.purge(command, command.route(), cleanup);
+                        Invariants.checkState(command.saveStatus() != SaveStatus.Uninitialised);
+                        List<Diff> arr = new ArrayList<>();
+                        arr.add(diff(null, command));
+                        updates.put(txnId, arr);
+                        break;
+                    case ERASE:
+                        removals.add(txnId);
+                        break;
+                }
+            }
+
+            for (TxnId removal : removals)
+                localJournal.remove(removal);
+            localJournal.putAll(updates);
         }
-        node.receive(request, from, replyContext);
     }
 
-    @Override
-    public void run()
+    // TODO (required): this might be a good first approximation, but maybe we need to make a better distinction between
+    // when we want to produce side-effects.
+    private boolean loading = false;
+
+    public void reconstructAll(InMemoryCommandStore.Loader loader, int commandStoreId)
     {
-        isScheduled = false;
-        if (this.node == null)
+        Map<TxnId, List<Diff>> diffs = diffsPerCommandStore.get(commandStoreId);
+
+        // Nothing to do here, journal is empty for this command store
+        if (diffs == null)
             return;
+
+        loading = true;
         try
         {
-            doRun();
+            List<Command> toApply = new ArrayList<>();
+            for (TxnId txnId : diffs.keySet())
+            {
+                Command command = reconstruct(commandStoreId, txnId);
+                if (command.saveStatus().compareTo(SaveStatus.Applying) >= 0 && !command.is(Invalidated) && !command.is(Truncated))
+                    toApply.add(command);
+                loader.load(command);
+            }
+
+            toApply.sort(Comparator.comparing(Command::executeAt));
+            for (Command command : toApply)
+                loader.apply(command);
         }
-        catch (Throwable t)
+        finally
         {
-            node.agent().onUncaughtException(t);
+            loading = false;
         }
     }
 
-    private void doRun()
+    private enum Reconstruct
     {
-        ArrayList<RequestContext> requests = null;
-        // check to see if any pending epochs are in
-        waitForEpochs.sort(null);
-        for (int i = 0; i < waitForEpochs.size(); i++)
-        {
-            long waitForEpoch = waitForEpochs.getLong(i);
-            if (!node.topology().hasEpoch(waitForEpoch))
-                break;
-            List<RequestContext> delayed = delayedRequests.remove(waitForEpoch);
-            if (null == requests) requests = new ArrayList<>(delayed.size());
-            requests.addAll(delayed);
-        }
-        waitForEpochs.removeIfLong(epoch -> !delayedRequests.containsKey(epoch));
+        Each,
+        Last
+    }
 
-        // for anything queued, put into the pending epochs or schedule
-        RequestContext request;
-        while (null != (request = unframedRequests.poll()))
-        {
-            long waitForEpoch = request.waitForEpoch;
-            if (waitForEpoch != 0 && !node.topology().hasEpoch(waitForEpoch))
-            {
-                delayedRequests.computeIfAbsent(waitForEpoch, ignore -> new ArrayList<>()).add(request);
-                if (!waitForEpochs.containsLong(waitForEpoch))
-                    waitForEpochs.addLong(waitForEpoch);
-            }
-            else
-            {
-                if (null == requests) requests = new ArrayList<>();
-                requests.add(request);
-            }
-        }
+    public void loadHistoricalTransactions(Consumer<Deps> consumer, int commandStoreId)
+    {
+        List<Deps> depsList = historicalTransactions.get(commandStoreId);
+        if (depsList == null)
+            return;
+        for (Deps deps : depsList)
+            consumer.accept(deps);
+    }
 
-        // schedule
-        if (requests != null)
-        {
-            requests.forEach(Runnable::run);
-        }
+    public Command reconstruct(int commandStoreId, TxnId txnId)
+    {
+        List<Diff> diffs = this.diffsPerCommandStore.get(commandStoreId).get(txnId);
+        return reconstruct(diffs, Reconstruct.Last).get(0);
     }
 
-    public Command reconstruct(int commandStoreId, TxnId txnId, Result result)
+    private List<Command> reconstruct(List<Diff> diffs, Reconstruct reconstruct)
     {
-        Key key = new Key(txnId, commandStoreId);
-        List<Diff> diffs = this.diffs.get(key);
-        if (diffs == null || diffs.isEmpty())
-            return null;
+        Invariants.checkState(diffs != null && !diffs.isEmpty());
+
+        List<Command> results = new ArrayList<>();
 
+        TxnId txnId = null;
         Timestamp executeAt = null;
         Timestamp executesAtLeast = null;
         SaveStatus saveStatus = null;
-        Status.Durability durability = null;
+        Status.Durability durability = Status.Durability.NotDurable;
 
         Ballot acceptedOrCommitted = Ballot.ZERO;
         Ballot promised = Ballot.ZERO;
@@ -173,94 +191,138 @@ public class Journal implements Runnable
 
         Command.WaitingOn waitingOn = null;
         Writes writes = null;
+        Result result = null;
 
-        for (Diff diff : diffs)
+        for (int i = 0; i < diffs.size(); i++)
         {
+            Diff diff = diffs.get(i);
             if (diff.txnId != null)
-                txnId = diff.txnId;
+                txnId = diff.txnId.get();
             if (diff.executeAt != null)
-                executeAt = diff.executeAt;
+                executeAt = diff.executeAt.get();
             if (diff.executesAtLeast != null)
-                executesAtLeast = diff.executesAtLeast;
+                executesAtLeast = diff.executesAtLeast.get();
             if (diff.saveStatus != null)
-                saveStatus = diff.saveStatus;
+            {
+                Set<SaveStatus> allowed = new HashSet<>();
+                allowed.add(SaveStatus.TruncatedApply);
+                allowed.add(SaveStatus.TruncatedApplyWithOutcome);
+
+                saveStatus = diff.saveStatus.get();
+            }
             if (diff.durability != null)
-                durability = diff.durability;
+                durability = diff.durability.get();
 
             if (diff.acceptedOrCommitted != null)
-                acceptedOrCommitted = diff.acceptedOrCommitted;
+                acceptedOrCommitted = diff.acceptedOrCommitted.get();
             if (diff.promised != null)
-                promised = diff.promised;
+                promised = diff.promised.get();
 
             if (diff.route != null)
-                route = diff.route;
+                route = diff.route.get();
             if (diff.partialTxn != null)
-                partialTxn = diff.partialTxn;
+                partialTxn = diff.partialTxn.get();
             if (diff.partialDeps != null)
-                partialDeps = diff.partialDeps;
+                partialDeps = diff.partialDeps.get();
             if (diff.additionalKeysOrRanges != null)
-                additionalKeysOrRanges = diff.additionalKeysOrRanges;
+                additionalKeysOrRanges = diff.additionalKeysOrRanges.get();
 
             if (diff.waitingOn != null)
-                waitingOn = diff.waitingOn;
+                waitingOn = diff.waitingOn.get();
             if (diff.writes != null)
-                writes = diff.writes;
-        }
+                writes = diff.writes.get();
 
-        if (!txnId.kind().awaitsOnlyDeps())
-            executesAtLeast = null;
+            if (diff.result != null)
+                result = diff.result.get();
 
-        switch (saveStatus.known.outcome)
-        {
-            case Erased:
-            case WasApply:
-                writes = null;
-                result = null;
-                break;
-        }
+            try
+            {
+                if (!txnId.kind().awaitsOnlyDeps())
+                    executesAtLeast = null;
+            }
+            catch (Throwable t)
+            {
+                t.printStackTrace();
+            }
+            switch (saveStatus.known.outcome)
+            {
+                case Erased:
+                case WasApply:
+                    writes = null;
+                    result = null;
+                    break;
+            }
 
-        CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(txnId);
-        if (partialTxn != null)
-            attrs.partialTxn(partialTxn);
-        if (durability != null)
-            attrs.durability(durability);
-        if (route != null)
-            attrs.route(route);
-
-        // TODO (desired): we can simplify this logic if, instead of diffing, we will infer the diff from the status
-        if (partialDeps != null &&
-            (saveStatus.known.deps != Status.KnownDeps.NoDeps &&
-             saveStatus.known.deps != Status.KnownDeps.DepsErased &&
-             saveStatus.known.deps != Status.KnownDeps.DepsUnknown))
-            attrs.partialDeps(partialDeps);
-        if (additionalKeysOrRanges != null)
-            attrs.additionalKeysOrRanges(additionalKeysOrRanges);
-        Invariants.checkState(saveStatus != null,
-                              "Save status is null after applying %s", diffs);
-
-        switch (saveStatus.status)
-        {
-            case NotDefined:
-                return saveStatus == SaveStatus.Uninitialised ? Command.NotDefined.uninitialised(attrs.txnId())
-                                                              : Command.NotDefined.notDefined(attrs, promised);
-            case PreAccepted:
-                return Command.PreAccepted.preAccepted(attrs, executeAt, promised);
-            case AcceptedInvalidate:
-            case Accepted:
-            case PreCommitted:
-                return Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted);
-            case Committed:
-            case Stable:
-                return Command.Committed.committed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn);
-            case PreApplied:
-            case Applied:
-                return Command.Executed.executed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn, writes, result);
-            case Invalidated:
-            case Truncated:
-                return truncated(attrs, saveStatus, executeAt, executesAtLeast, writes, result);
-            default:
-                throw new IllegalStateException("Do not know " + saveStatus.status + " " + saveStatus);
+            CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(txnId);
+            if (partialTxn != null)
+                attrs.partialTxn(partialTxn);
+            if (durability != null)
+                attrs.durability(durability);
+            if (route != null)
+                attrs.route(route);
+
+            // TODO (desired): we can simplify this logic if, instead of diffing, we will infer the diff from the status
+            if (partialDeps != null &&
+                (saveStatus.known.deps != Status.KnownDeps.NoDeps &&
+                 saveStatus.known.deps != Status.KnownDeps.DepsErased &&
+                 saveStatus.known.deps != Status.KnownDeps.DepsUnknown))
+                attrs.partialDeps(partialDeps);
+            if (additionalKeysOrRanges != null)
+                attrs.additionalKeysOrRanges(additionalKeysOrRanges);
+            Invariants.checkState(saveStatus != null,
+                                  "Save status is null after applying %s", diffs);
+
+            try
+            {
+                if (reconstruct == Reconstruct.Each ||
+                    (reconstruct == Reconstruct.Last && i == diffs.size() - 1))
+                {
+                    Command current;
+                    switch (saveStatus.status)
+                    {
+                        case NotDefined:
+                            current = saveStatus == SaveStatus.Uninitialised ? Command.NotDefined.uninitialised(attrs.txnId())
+                                                                             : Command.NotDefined.notDefined(attrs, promised);
+                            break;
+                        case PreAccepted:
+                            current = Command.PreAccepted.preAccepted(attrs, executeAt, promised);
+                            break;
+                        case AcceptedInvalidate:
+                        case Accepted:
+                        case PreCommitted:
+                            if (saveStatus == SaveStatus.AcceptedInvalidate)
+                                current = Command.AcceptedInvalidateWithoutDefinition.acceptedInvalidate(attrs, promised, acceptedOrCommitted);
+                            else
+                                current = Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted);
+                            break;
+                        case Committed:
+                        case Stable:
+                            current = Command.Committed.committed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn);
+                            break;
+                        case PreApplied:
+                        case Applied:
+                            current = Command.Executed.executed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn, writes, result);
+                            break;
+                        case Invalidated:
+                        case Truncated:
+                            current = truncated(attrs, saveStatus, executeAt, executesAtLeast, writes, result);
+                            break;
+                        default:
+                            throw new IllegalStateException("Do not know " + saveStatus.status + " " + saveStatus);
+                    }
+
+                    results.add(current);
+                }
+            }
+            catch (Throwable t)
+            {
+
+                throw new RuntimeException("Can not reconstruct from diff:\n" + diffs.stream().map(o -> o.toString())
+                                                                                     .collect(Collectors.joining("\n")),
+                                           t);
+            }
         }
+        return results;
     }
 
     private static Command.Truncated truncated(CommonAttributes.Mutable attrs, SaveStatus status, Timestamp executeAt, Timestamp executesAtLeast, Writes writes, Result result)
@@ -282,77 +344,74 @@ public class Journal implements Runnable
         }
     }
 
-    public void onExecute(int commandStoreId, Command before, Command after, boolean isPrimary)
+    public void registerHistoricalTransactions(int commandStoreId, Deps deps)
     {
-        if (before == null && after == null)
-            return;
-        Diff diff = diff(before, after);
-        if (!isPrimary)
-            diff = diff.asNonprimary();
-
-        if (diff != null)
-        {
-            Key key = new Key(after.txnId(), commandStoreId);
-            diffs.computeIfAbsent(key, (k_) -> new ArrayList<>()).add(diff);
-        }
+        this.historicalTransactions.computeIfAbsent(commandStoreId, (k) -> new ArrayList<>()).add(deps);
     }
 
-    private static class RequestContext implements Runnable
+    public void onExecute(int commandStoreId, Command before, Command after, boolean isPrimary)
     {
-        final long waitForEpoch;
-        final Message message;
-        final Runnable fn;
+        if (loading || (before == null && after == null))
+            return;
 
-        protected RequestContext(Message request, long waitForEpoch, Runnable fn)
+        if (after.saveStatus() == SaveStatus.Erased)
         {
-            this.waitForEpoch = waitForEpoch;
-            this.message = request;
-            this.fn = fn;
+            diffsPerCommandStore.computeIfAbsent(commandStoreId, (k) -> new TreeMap<>())
+                                .remove(after.txnId());
+            return;
         }
+        Diff diff = diff(before, after);
+        if (!isPrimary)
+            diff = diff.asNonPrimary();
 
-        @Override
-        public void run()
+        if (diff != null)
         {
-            fn.run();
+            diffsPerCommandStore.computeIfAbsent(commandStoreId, (k) -> new TreeMap<>())
+                                .computeIfAbsent(after.txnId(), (k_) -> new ArrayList<>())
+                                .add(diff);
         }
     }
 
     private static class Diff
     {
-        public final TxnId txnId;
+        public final NewValue<TxnId> txnId;
 
-        public final Timestamp executeAt;
-        public final Timestamp executesAtLeast;
-        public final SaveStatus saveStatus;
-        public final Status.Durability durability;
+        public final NewValue<Timestamp> executeAt;
+        public final NewValue<Timestamp> executesAtLeast;
+        public final NewValue<SaveStatus> saveStatus;
+        public final NewValue<Status.Durability> durability;
 
-        public final Ballot acceptedOrCommitted;
-        public final Ballot promised;
+        public final NewValue<Ballot> acceptedOrCommitted;
+        public final NewValue<Ballot> promised;
 
-        public final Route<?> route;
-        public final PartialTxn partialTxn;
-        public final PartialDeps partialDeps;
+        public final NewValue<Route<?>> route;
+        public final NewValue<PartialTxn> partialTxn;
+        public final NewValue<PartialDeps> partialDeps;
 
-        public final Writes writes;
-        public final Command.WaitingOn waitingOn;
-        public final Seekables<?, ?> additionalKeysOrRanges;
+        public final NewValue<Writes> writes;
+        public final NewValue<Command.WaitingOn> waitingOn;
+        public final NewValue<Seekables<?, ?>> additionalKeysOrRanges;
 
-        public Diff(TxnId txnId,
-                    Timestamp executeAt,
-                    Timestamp executesAtLeast,
-                    SaveStatus saveStatus,
-                    Status.Durability durability,
+        public final NewValue<Result> result; // temporarily here for sakes for reloads
 
-                    Ballot acceptedOrCommitted,
-                    Ballot promised,
+        public Diff(NewValue<TxnId> txnId,
+                    NewValue<Timestamp> executeAt,
+                    NewValue<Timestamp> executesAtLeast,
+                    NewValue<SaveStatus> saveStatus,
+                    NewValue<Status.Durability> durability,
 
-                    Route<?> route,
-                    PartialTxn partialTxn,
-                    PartialDeps partialDeps,
-                    Command.WaitingOn waitingOn,
+                    NewValue<Ballot> acceptedOrCommitted,
+                    NewValue<Ballot> promised,
 
-                    Writes writes,
-                    Seekables<?, ?> additionalKeysOrRanges)
+                    NewValue<Route<?>> route,
+                    NewValue<PartialTxn> partialTxn,
+                    NewValue<PartialDeps> partialDeps,
+                    NewValue<Command.WaitingOn> waitingOn,
+
+                    NewValue<Writes> writes,
+                    NewValue<Seekables<?, ?>> additionalKeysOrRanges,
+
+                    NewValue<Result> result)
         {
             this.txnId = txnId;
             this.executeAt = executeAt;
@@ -370,12 +429,13 @@ public class Journal implements Runnable
             this.writes = writes;
             this.waitingOn = waitingOn;
             this.additionalKeysOrRanges = additionalKeysOrRanges;
+            this.result = result;
         }
 
-        // We allow only save status, waitingOn, and listeners to be updated by non-primary transactions
-        public Diff asNonprimary()
+        // We allow only save status, and waitingOn to be updated by non-primary transactions
+        public Diff asNonPrimary()
         {
-            return new Diff(null, null, null, saveStatus, null, null, null, null, null, null, waitingOn, null, null);
+            return new Diff(txnId, null, null, saveStatus, null, null, null, null, null, null, waitingOn, null, additionalKeysOrRanges, null);
         }
 
         public boolean allNulls()
@@ -393,13 +453,14 @@ public class Journal implements Runnable
             if (writes != null) return false;
             if (waitingOn != null) return false;
             if (additionalKeysOrRanges != null) return false;
+            if (result != null) return false;
             return true;
         }
 
         @Override
         public String toString()
         {
-            StringBuilder builder = new StringBuilder("SavedDiff{");
+            StringBuilder builder = new StringBuilder("Diff{");
             if (txnId != null)
                 builder.append("txnId = ").append(txnId).append(" ");
             if (executeAt != null)
@@ -426,6 +487,8 @@ public class Journal implements Runnable
                 builder.append("waitingOn = ").append(waitingOn).append(" ");
             if (additionalKeysOrRanges != null)
                 builder.append("additionalKeysOrRanges = ").append(additionalKeysOrRanges).append(" ");
+            if (result != null)
+                builder.append("result = ").append(result).append(" ");
             builder.append("}");
             return builder.toString();
         }
@@ -450,7 +513,9 @@ public class Journal implements Runnable
                              ifNotEqual(before, after, Command::partialDeps, false),
                              ifNotEqual(before, after, Journal::getWaitingOn, true),
                              ifNotEqual(before, after, Command::writes, false),
-                             ifNotEqual(before, after, Command::additionalKeysOrRanges, false));
+                             ifNotEqual(before, after, Command::additionalKeysOrRanges, false),
+                             ifNotEqual(before, after, Command::result, false));
+
         if (diff.allNulls())
             return null;
 
@@ -465,7 +530,7 @@ public class Journal implements Runnable
         return null;
     }
 
-    private static <OBJ, VAL> VAL ifNotEqual(OBJ lo, OBJ ro, Function<OBJ, VAL> convert, boolean allowClassMismatch)
+    private static <OBJ, VAL> NewValue<VAL> ifNotEqual(OBJ lo, OBJ ro, Function<OBJ, VAL> convert, boolean allowClassMismatch)
     {
         VAL l = null;
         VAL r = null;
@@ -473,39 +538,42 @@ public class Journal implements Runnable
         if (ro != null) r = convert.apply(ro);
 
         if (l == r)
-            return null;
+            return null; // null here means there was no change
+
         if (l == null || r == null)
-            return r;
+            return NewValue.of(r);
+
         assert allowClassMismatch || l.getClass() == r.getClass() : String.format("%s != %s", l.getClass(), r.getClass());
 
         if (l.equals(r))
             return null;
 
-        return r;
+        return NewValue.of(r);
     }
 
-    public static class Key
+    private static class NewValue<T>
     {
-        final TxnId timestamp;
-        final int commandStoreId;
+        final T value;
 
-        public Key(TxnId timestamp, int commandStoreId)
+        private NewValue(T value)
         {
-            this.timestamp = timestamp;
-            this.commandStoreId = commandStoreId;
+            this.value = value;
         }
 
-        public boolean equals(Object o)
+        public T get()
         {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Key key = (Key) o;
-            return commandStoreId == key.commandStoreId && Objects.equals(timestamp, key.timestamp);
+            return value;
         }
 
-        public int hashCode()
+        public static <T> NewValue<T> of(T value)
         {
-            return Objects.hash(timestamp, commandStoreId);
+            return new NewValue<>(value);
+        }
+
+        public String toString()
+        {
+            return "" + value;
         }
     }
+
 }
\ No newline at end of file
diff --git a/accord-core/src/test/java/accord/impl/basic/NodeSink.java b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
index e3a95775..bfa4b3c9 100644
--- a/accord-core/src/test/java/accord/impl/basic/NodeSink.java
+++ b/accord-core/src/test/java/accord/impl/basic/NodeSink.java
@@ -22,6 +22,9 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.function.Function;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import accord.api.MessageSink;
 import accord.impl.basic.Cluster.Link;
 import accord.local.AgentExecutor;
@@ -41,6 +44,8 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 public class NodeSink implements MessageSink
 {
+    private static final Logger logger = LoggerFactory.getLogger(NodeSink.class);
+
     private static final boolean DEBUG = false;
     public enum Action { DELIVER, DROP, DELIVER_WITH_FAILURE, FAILURE }
 
@@ -150,8 +155,8 @@ public class NodeSink implements MessageSink
     }
 
     @Override
-    public void replyWithUnknownFailure(Id replyingToNode, ReplyContext replyContext, Throwable failure)
+    public void replyWithUnknownFailure(Id replyToNode, ReplyContext replyContext, Throwable failure)
     {
-        reply(replyingToNode, replyContext, new FailureReply(failure));
+        reply(replyToNode, replyContext, new FailureReply(failure));
     }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java b/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
index fd4932d4..15c75143 100644
--- a/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/PendingQueue.java
@@ -18,15 +18,20 @@
 
 package accord.impl.basic;
 
+import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 
-public interface PendingQueue
+public interface PendingQueue extends Iterable<Pending>
 {
     void add(Pending item);
     void addNoDelay(Pending item);
     void add(Pending item, long delay, TimeUnit units);
     boolean remove(Pending item);
     Pending poll();
+    List<Pending> drain(Predicate<Pending> toDrain);
     int size();
     long nowInMillis();
+
+    boolean hasNonRecurring();
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java b/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
index bd25ac8f..430adbb1 100644
--- a/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/PropagatingPendingQueue.java
@@ -18,8 +18,10 @@
 
 package accord.impl.basic;
 
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 
 public class PropagatingPendingQueue implements PendingQueue
 {
@@ -63,6 +65,12 @@ public class PropagatingPendingQueue implements PendingQueue
         return wrapped.poll();
     }
 
+    @Override
+    public List<Pending> drain(Predicate<Pending> toDrain)
+    {
+        return wrapped.drain(toDrain);
+    }
+
     public void checkFailures()
     {
         if (!failures.isEmpty())
@@ -99,4 +107,16 @@ public class PropagatingPendingQueue implements PendingQueue
     {
         return wrapped.nowInMillis();
     }
+
+    @Override
+    public boolean hasNonRecurring()
+    {
+        return wrapped.hasNonRecurring();
+    }
+
+    @Override
+    public Iterator<Pending> iterator()
+    {
+        return wrapped.iterator();
+    }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
index 1468719b..e568e5c4 100644
--- a/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
+++ b/accord-core/src/test/java/accord/impl/basic/RandomDelayQueue.java
@@ -18,14 +18,18 @@
 
 package accord.impl.basic;
 
-import accord.burn.random.FrequentLargeRange;
-import accord.utils.RandomSource;
-
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 import java.util.PriorityQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.function.LongSupplier;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
+import accord.burn.random.FrequentLargeRange;
+import accord.utils.RandomSource;
+
 import static accord.utils.Invariants.illegalArgument;
 
 public class RandomDelayQueue implements PendingQueue
@@ -94,6 +98,7 @@ public class RandomDelayQueue implements PendingQueue
     private final LongSupplier jitterMillis;
     long now;
     int seq;
+    int recurring;
 
     public RandomDelayQueue(RandomSource random)
     {
@@ -115,6 +120,8 @@ public class RandomDelayQueue implements PendingQueue
     public void addNoDelay(Pending item)
     {
         queue.add(new Item(now, seq++, item));
+        if (item instanceof RecurringPendingRunnable)
+            ++recurring;
     }
 
     @Override
@@ -123,11 +130,15 @@ public class RandomDelayQueue implements PendingQueue
         if (delay < 0)
             throw illegalArgument("Delay must be positive or 0, but given " + delay);
         queue.add(new Item(now + units.toMillis(delay) + jitterMillis.getAsLong(), seq++, item));
+        if (item instanceof RecurringPendingRunnable)
+            ++recurring;
     }
 
     @Override
     public boolean remove(Pending item)
     {
+        if (item instanceof RecurringPendingRunnable)
+            --recurring;
         return queue.remove(item);
     }
 
@@ -137,10 +148,35 @@ public class RandomDelayQueue implements PendingQueue
         Item item = queue.poll();
         if (item == null)
             return null;
+
+        if (item.item instanceof RecurringPendingRunnable)
+            --recurring;
+
         now = item.time;
         return item.item;
     }
 
+    @Override
+    public List<Pending> drain(Predicate<Pending> toDrain)
+    {
+        List<Item> items = new ArrayList<>(queue);
+        queue.clear();
+        recurring = 0;
+        List<Pending> ret = new ArrayList<>();
+        for (Item item : items)
+        {
+            if (toDrain.test(item.item))
+                ret.add(item.item);
+            else
+            {
+                queue.add(item);
+                if (item.item instanceof RecurringPendingRunnable)
+                    ++recurring;
+            }
+        }
+        return ret;
+    }
+
     @Override
     public int size()
     {
@@ -152,4 +188,16 @@ public class RandomDelayQueue implements PendingQueue
     {
         return now;
     }
+
+    @Override
+    public boolean hasNonRecurring()
+    {
+        return recurring != queue.size();
+    }
+
+    @Override
+    public Iterator<Pending> iterator()
+    {
+        return queue.stream().map(i -> i.item).iterator();
+    }
 }
diff --git a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
index 8dd2e3d2..d8b876a0 100644
--- a/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
+++ b/accord-core/src/test/java/accord/impl/basic/RecurringPendingRunnable.java
@@ -19,18 +19,19 @@
 package accord.impl.basic;
 
 import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
 
 import accord.api.Scheduler.Scheduled;
 
 class RecurringPendingRunnable implements PendingRunnable, Scheduled
 {
     final PendingQueue requeue;
-    final long delay;
+    final LongSupplier delay;
     final TimeUnit units;
     Runnable run;
     Runnable onCancellation;
 
-    RecurringPendingRunnable(PendingQueue requeue, Runnable run, long delay, TimeUnit units)
+    RecurringPendingRunnable(PendingQueue requeue, Runnable run, LongSupplier delay, TimeUnit units)
     {
         this.requeue = requeue;
         this.run = run;
@@ -44,11 +45,16 @@ class RecurringPendingRunnable implements PendingRunnable, Scheduled
         if (run != null)
         {
             run.run();
-            if (requeue != null) requeue.add(this, delay, units);
-            else run = null;
+            maybeRequeue();
         }
     }
 
+    public void maybeRequeue()
+    {
+        if (requeue != null) requeue.add(this, delay.getAsLong(), units);
+        else run = null;
+    }
+
     @Override
     public void cancel()
     {
diff --git a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
index a4687514..c7ffc2c6 100644
--- a/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
+++ b/accord-core/src/test/java/accord/impl/list/ListFetchCoordinator.java
@@ -65,7 +65,7 @@ public class ListFetchCoordinator extends AbstractFetchCoordinator
         ListData listData = (ListData) data;
         persisting.add(commandStore.execute(PreLoadContext.empty(), safeStore -> {
             listData.forEach((key, value) -> listStore.data.merge(key, value, Timestamped::merge));
-        }).addCallback((ignore, fail) -> {
+        }).flatMap(ignore -> listStore.snapshot(received, syncPoint.syncId)).addCallback((success, fail) -> {
             if (fail == null) success(from, received);
             else fail(from, received, fail);
         }).beginAsResult());
diff --git a/accord-core/src/test/java/accord/impl/list/ListStore.java b/accord-core/src/test/java/accord/impl/list/ListStore.java
index cbcdd89c..71aecfde 100644
--- a/accord-core/src/test/java/accord/impl/list/ListStore.java
+++ b/accord-core/src/test/java/accord/impl/list/ListStore.java
@@ -18,8 +18,10 @@
 
 package accord.impl.list;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Deque;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -30,6 +32,7 @@ import java.util.stream.Collectors;
 
 import accord.api.DataStore;
 import accord.api.Key;
+import accord.api.Scheduler;
 import accord.coordinate.CoordinateSyncPoint;
 import accord.coordinate.ExecuteSyncPoint.SyncPointErased;
 import accord.coordinate.Invalidated;
@@ -54,9 +57,12 @@ import accord.primitives.Timestamp;
 import accord.primitives.TxnId;
 import accord.topology.Topologies;
 import accord.topology.Topology;
+import accord.utils.Invariants;
+import accord.utils.RandomSource;
 import accord.utils.Timestamped;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 import org.agrona.collections.Int2ObjectHashMap;
 import org.agrona.collections.LongArrayList;
@@ -136,6 +142,9 @@ public class ListStore implements DataStore
 
     static final Timestamped<int[]> EMPTY = new Timestamped<>(Timestamp.NONE, new int[0], Arrays::toString);
     final NavigableMap<RoutableKey, Timestamped<int[]>> data = new TreeMap<>();
+    final Scheduler scheduler;
+    final RandomSource random;
+
     private final List<ChangeAt> addedAts = new ArrayList<>();
     private final List<ChangeAt> removedAts = new ArrayList<>();
     private final List<PurgeAt> purgedAts = new ArrayList<>();
@@ -148,11 +157,104 @@ public class ListStore implements DataStore
     // when out of order epochs are detected, this holds the callbacks to try again
     private final List<Runnable> onRemovalDone = new ArrayList<>();
 
+    private static final class Snapshot
+    {
+        private final NavigableMap<RoutableKey, Timestamped<int[]>> data;
+        private final List<ChangeAt> addedAts;
+        private final List<ChangeAt> removedAts;
+        private final List<PurgeAt> purgedAts;
+        private final List<FetchComplete> fetchCompletes;
+        private final LongArrayList pendingRemoves;
+
+        private Snapshot(NavigableMap<RoutableKey, Timestamped<int[]>> data, List<ChangeAt> addedAts, List<ChangeAt> removedAts, List<PurgeAt> purgedAts, List<FetchComplete> fetchCompletes, LongArrayList pendingRemoves)
+        {
+            this.data = new TreeMap<>(data);
+            this.addedAts = new ArrayList<>(addedAts);
+            this.removedAts = new ArrayList<>(removedAts);
+            this.purgedAts = new ArrayList<>(purgedAts);
+            this.fetchCompletes = new ArrayList<>(fetchCompletes);
+            this.pendingRemoves = new LongArrayList();
+            this.pendingRemoves.addAll(pendingRemoves);
+        }
+    }
+
+    private static final class PendingSnapshot
+    {
+        final long delay;
+        final Runnable runnable;
+
+        private PendingSnapshot(long delay, Runnable runnable)
+        {
+            this.delay = delay;
+            this.runnable = runnable;
+        }
+    }
+
+    private Snapshot snapshot;
+    private final Deque<PendingSnapshot> pendingSnapshots = new ArrayDeque<>();
+    private long pendingDelay = 0;
+
+    public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+    {
+        Snapshot snapshot = new Snapshot(data, addedAts, removedAts, purgedAts, fetchCompletes, pendingRemoves);
+        AsyncResult.Settable<Void> result = new AsyncResults.SettableResult<>();
+        long delay = Math.max(1, random.nextBiasedLong(100, 1000, 5000) - pendingDelay);
+        pendingDelay += delay;
+        pendingSnapshots.add(new PendingSnapshot(delay, () -> {
+            this.snapshot = snapshot;
+            result.setSuccess(null);
+        }));
+
+        if (pendingSnapshots.size() == 1)
+            scheduleRunSnapshot();
+        return result;
+    }
+
+    private void scheduleRunSnapshot()
+    {
+        Invariants.checkState(!pendingSnapshots.isEmpty());
+        scheduler.once(() -> {
+            if (pendingSnapshots.isEmpty())
+                return;
+
+            PendingSnapshot pendingSnapshot = pendingSnapshots.pollFirst();
+            pendingSnapshot.runnable.run();
+            pendingDelay -= pendingSnapshot.delay;
+            if (!pendingSnapshots.isEmpty())
+                scheduleRunSnapshot();
+        }, pendingSnapshots.peekFirst().delay, TimeUnit.MILLISECONDS);
+    }
+
+    public void restoreFromSnapshot()
+    {
+        if (snapshot == null)
+            return;
+
+        data.putAll(snapshot.data);
+        addedAts.addAll(snapshot.addedAts);
+        removedAts.addAll(snapshot.removedAts);
+        purgedAts.addAll(snapshot.purgedAts);
+        fetchCompletes.addAll(snapshot.fetchCompletes);
+        pendingRemoves.addAll(snapshot.pendingRemoves);
+    }
+
+    public void clear()
+    {
+        data.clear();
+        addedAts.clear();
+        removedAts.clear();
+        purgedAts.clear();
+        fetchCompletes.clear();
+        pendingRemoves.clear();
+    }
+
     // adding here to help trace burn test queries
     public final Node.Id node;
 
-    public ListStore(Node.Id node)
+    public ListStore(Scheduler scheduler, RandomSource random, Node.Id node)
     {
+        this.scheduler = scheduler;
+        this.random = random;
         this.node = node;
     }
 
diff --git a/accord-core/src/test/java/accord/impl/mock/MockStore.java b/accord-core/src/test/java/accord/impl/mock/MockStore.java
index e3613fb2..803c7efc 100644
--- a/accord-core/src/test/java/accord/impl/mock/MockStore.java
+++ b/accord-core/src/test/java/accord/impl/mock/MockStore.java
@@ -33,9 +33,11 @@ import accord.primitives.Seekable;
 import accord.primitives.Seekables;
 import accord.primitives.SyncPoint;
 import accord.primitives.Timestamp;
+import accord.primitives.TxnId;
 import accord.primitives.Writes;
 import accord.utils.async.AsyncChain;
 import accord.utils.async.AsyncChains;
+import accord.utils.async.AsyncResult;
 import accord.utils.async.AsyncResults;
 
 public class MockStore implements DataStore
@@ -149,4 +151,10 @@ public class MockStore implements DataStore
         callback.fetched(ranges);
         return new ImmediateFetchFuture(ranges);
     }
+
+    @Override
+    public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+    {
+        return AsyncResults.success(null);
+    }
 }
diff --git a/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java b/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java
deleted file mode 100644
index 4d598d9e..00000000
--- a/accord-core/src/test/java/accord/local/BootstrapLocalTxnTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package accord.local;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Consumer;
-
-import org.junit.jupiter.api.Test;
-
-import accord.impl.PrefixedIntHashKey;
-import accord.impl.basic.Cluster;
-import accord.impl.basic.DelayedCommandStores.DelayedCommandStore;
-import accord.messages.MessageType;
-import accord.messages.ReplyContext;
-import accord.messages.Request;
-import accord.primitives.Deps;
-import accord.primitives.FullRoute;
-import accord.primitives.Ranges;
-import accord.primitives.Routable;
-import accord.primitives.SyncPoint;
-import accord.primitives.Txn;
-import accord.primitives.TxnId;
-import accord.topology.Topology;
-import accord.topology.TopologyUtils;
-import accord.utils.AccordGens;
-import accord.utils.Gen;
-import accord.utils.Gens;
-import accord.utils.Invariants;
-import org.assertj.core.api.Assertions;
-
-import static accord.local.PreLoadContext.contextFor;
-import static accord.utils.Property.qt;
-
-class BootstrapLocalTxnTest
-{
-    private static final Gen<Gen<Cleanup>> CLEANUP_DISTRIBUTION = Gens.mixedDistribution(Cleanup.NO, Cleanup.TRUNCATE, Cleanup.TRUNCATE_WITH_OUTCOME, Cleanup.ERASE);
-
-    @Test
-    public void localOnlyTxnLifeCycle()
-    {
-        Ranges ranges = Ranges.ofSortedAndDeoverlapped(PrefixedIntHashKey.ranges(0, 1));
-        List<Node.Id> nodes = Collections.singletonList(new Node.Id(42));
-        Topology t = TopologyUtils.topology(1, nodes, ranges, 2);
-        qt().check(rs -> Cluster.run(rs::fork, nodes, t, nodeMap -> new Request()
-        {
-            @Override
-            public void process(Node on, Node.Id from, ReplyContext replyContext)
-            {
-                Gen<Cleanup> cleanupGen = CLEANUP_DISTRIBUTION.next(rs);
-                for (int storeId : on.commandStores().ids())
-                {
-                    DelayedCommandStore store = (DelayedCommandStore) on.commandStores().forId(storeId);
-                    // this is a bit redudent but here to make the test easier to maintain.  Pre/Post execute we validate each command to make sure everything is fine
-                    // but that logic could be changed and this test has a dependency on validation the command... so to make this dependency explicit
-                    // the test will call the validation logic within the test even though it will be called again in the background...
-                    Consumer<Command> validate = store::validateRead;
-                    TxnId globalSyncId = on.nextTxnId(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range);
-                    TxnId localSyncId = globalSyncId.as(Txn.Kind.LocalOnly);
-                    TxnId nextGlobalSyncId = on.nextTxnId(Txn.Kind.ExclusiveSyncPoint, Routable.Domain.Range).withEpoch(globalSyncId.epoch() + 1);
-                    Ranges ranges = AccordGens.rangesInsideRanges(store.updateRangesForEpoch().currentRanges(), (rs2, r) -> rs2.nextInt(1, 4)).next(rs);
-
-                    FullRoute<?> route = on.computeRoute(globalSyncId, ranges);
-                    SyncPoint<Ranges> syncPoint = new SyncPoint<>(globalSyncId, Deps.NONE, ranges, route);
-                    Ranges valid = AccordGens.rangesInsideRanges(ranges, (rs2, r) -> rs2.nextInt(1, 4)).next(rs);
-                    Invariants.checkArgument(syncPoint.keysOrRanges.containsAll(valid));
-                    store.execute(contextFor(localSyncId, syncPoint.waitFor.keyDeps.keys(), KeyHistory.COMMANDS), safe -> Commands.createBootstrapCompleteMarkerTransaction(safe, localSyncId, valid))
-                         .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> validate.accept(safe.get(localSyncId, route.homeKey()).current())))
-                         .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> Commands.markBootstrapComplete(safe, localSyncId, ranges)))
-                         .flatMap(ignore -> store.execute(contextFor(localSyncId), safe -> validate.accept(safe.get(localSyncId, route.homeKey()).current())))
-                         // cleanup txn
-                         .flatMap(ignore -> store.submit(PreLoadContext.empty(), safe -> {
-                             Cleanup target = cleanupGen.next(rs);
-                             if (target == Cleanup.NO)
-                                 return Cleanup.NO;
-                             safe.commandStore().setRedundantBefore(RedundantBefore.create(ranges, Long.MIN_VALUE, Long.MAX_VALUE, nextGlobalSyncId, nextGlobalSyncId, TxnId.NONE));
-                             switch (target)
-                             {
-                                 case ERASE:
-                                     safe.commandStore().setDurableBefore(DurableBefore.create(ranges, nextGlobalSyncId, nextGlobalSyncId));
-                                     break;
-                                 case TRUNCATE:
-                                     safe.commandStore().setDurableBefore(DurableBefore.create(ranges, nextGlobalSyncId, globalSyncId));
-                                     break;
-                                 case TRUNCATE_WITH_OUTCOME:
-                                 case INVALIDATE:
-                                     // no update to DurableBefore = TRUNCATE_WITH_OUTCOME
-                                     break;
-                                 default:
-                                     throw new UnsupportedOperationException(target.name());
-                             }
-                             return target;
-                         }))
-                         // validateRead is called implicitly _on command completion_
-                         .flatMap(target -> store.execute(contextFor(localSyncId), safe -> {
-                             SafeCommand cmd = safe.get(localSyncId, route.homeKey());
-                             Command current = cmd.current();
-                             Assertions.assertThat(current.saveStatus()).isEqualTo(target == Cleanup.NO ? SaveStatus.Applied : target.appliesIfNot);
-                         }))
-                         .begin(on.agent());
-                }
-            }
-
-            @Override
-            public MessageType type()
-            {
-                return null;
-            }
-        }));
-    }
-}
\ No newline at end of file
diff --git a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
index 271037ad..6b7c0b09 100644
--- a/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
+++ b/accord-core/src/test/java/accord/local/cfk/CommandsForKeyTest.java
@@ -47,10 +47,10 @@ import accord.api.Read;
 import accord.api.Result;
 import accord.api.RoutingKey;
 import accord.api.Update;
-import accord.impl.IntKey;
 import accord.impl.DefaultLocalListeners;
 import accord.impl.DefaultLocalListeners.DefaultNotifySink;
 import accord.impl.DefaultRemoteListeners;
+import accord.impl.IntKey;
 import accord.local.Command;
 import accord.local.Command.AbstractCommand;
 import accord.local.CommandStore;
@@ -226,12 +226,6 @@ public class CommandsForKeyTest
             set(prev, next);
         }
 
-        @Override
-        public void notWaiting(SafeCommandStore safeStore, SafeCommand safeCommand, Key key)
-        {
-            notWaiting(safeStore, safeCommand.txnId(), key);
-        }
-
         @Override
         public void waitingOn(SafeCommandStore safeStore, TxnInfo txn, Key key, SaveStatus waitingOnStatus, BlockedUntil blockedUntil, boolean notifyCfk)
         {
@@ -837,12 +831,6 @@ public class CommandsForKeyTest
         {
             throw new UnsupportedOperationException();
         }
-
-        @Override
-        public void registerHistoricalTransactions(Deps deps)
-        {
-            throw new UnsupportedOperationException();
-        }
     }
 
     private static final class TestRead implements Read
@@ -915,7 +903,13 @@ public class CommandsForKeyTest
 
         protected TestCommandStore(int pruneInterval, int pruneHlcDelta)
         {
-            super(0, null, null, null, ignore -> new ProgressLog.NoOpProgressLog(), ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultNotifySink.INSTANCE), new EpochUpdateHolder());
+            super(0,
+                  null,
+                  null,
+                  null,
+                  ignore -> new ProgressLog.NoOpProgressLog(),
+                  ignore -> new DefaultLocalListeners(new DefaultRemoteListeners((a, b, c, d, e)->{}), DefaultNotifySink.INSTANCE),
+                  new EpochUpdateHolder());
             this.pruneInterval = pruneInterval;
             this.pruneHlcDelta = pruneHlcDelta;
         }
@@ -961,6 +955,12 @@ public class CommandsForKeyTest
             Invariants.checkState(queue.isEmpty());
         }
 
+        @Override
+        protected void registerHistoricalTransactions(Deps deps, SafeCommandStore safeStore)
+        {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public <T> AsyncChain<T> submit(Callable<T> task)
         {
diff --git a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
index 4faa4cc2..c0cb6b95 100644
--- a/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
+++ b/accord-core/src/test/java/accord/verify/StrictSerializabilityVerifier.java
@@ -460,6 +460,8 @@ public class StrictSerializabilityVerifier implements Verifier
                     .append(", preds:").append(Arrays.toString(maxPredecessors))
                     .append(unknownStepPeers != null ? ", peers?:" + unknownStepPeers : "")
                     .append(unknownStepPredecessors != null ? ", preds?:" + unknownStepPredecessors.values() : "")
+                   .append(" writtenBefore " + writtenBefore)
+                   .append(" writtenAfter " + writtenAfter)
                     .append("}");
             return builder.toString();
         }
@@ -709,7 +711,7 @@ public class StrictSerializabilityVerifier implements Verifier
                 throw new HistoryViolation(prefix, key, "Cycle detected on key " + prefix + ":" + key + ", step " + step.ofStepIndex + " " + Arrays.toString(Arrays.copyOf(sequence, step.ofStepIndex)));
 
             if (step.writtenBefore < step.writtenAfter)
-                throw new HistoryViolation(prefix, key, prefix + ":" + key + ": timestamp inconsistency, step " + step.ofStepIndex);
+                throw new HistoryViolation(prefix, key, String.format("%s:%d: timestamp inconsistency, step %d. %s", prefix, key, step.ofStepIndex, step));
 
             if (step.maxPredecessorWrittenAfter > step.writtenBefore)
                 throw new HistoryViolation(prefix, key, prefix + ":" + key + " must have been written prior to its maximum predecessor in real-time order on step " + step.ofStepIndex);
diff --git a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
index 013db36d..777700cc 100644
--- a/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
+++ b/accord-maelstrom/src/main/java/accord/maelstrom/MaelstromStore.java
@@ -27,7 +27,10 @@ import accord.local.Node;
 import accord.local.SafeCommandStore;
 import accord.primitives.Ranges;
 import accord.primitives.SyncPoint;
+import accord.primitives.TxnId;
 import accord.utils.Timestamped;
+import accord.utils.async.AsyncResult;
+import accord.utils.async.AsyncResults;
 import accord.utils.async.AsyncResults.SettableResult;
 
 public class MaelstromStore implements DataStore
@@ -57,4 +60,10 @@ public class MaelstromStore implements DataStore
     {
         return new ImmediateFetchResult(ranges);
     }
+
+    @Override
+    public AsyncResult<Void> snapshot(Ranges ranges, TxnId before)
+    {
+        return AsyncResults.success(null);
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org